Fix current jQuery version

v1.13

.bowerrc

@@ -1,3 +0,0 @@
-{
-  "directory" : "bower_components"
-}

.gitignore

@@ -1,5 +1,4 @@
 node_modules
-bower_components
 .tmp
 .vagrant
 results/*

.travis.yml

@@ -1,8 +1,19 @@
 language: node_js
+sudo: false
 node_js:
-    - "0.12.4"
+  - "8.9"
+  - "6.2"
+  - "4.7"
+env:
+  - CXX=g++-4.8
+addons:
+  apt:
+    sources:
+      - ubuntu-toolchain-r-test
+    packages:
+      - g++-4.8
 before_install:
-    - "npm install -g npm"
-    - "npm install -g grunt-cli"
+  - "npm install -g npm"
+  - "npm install -g grunt-cli"
 install:
-    - "npm install"
+  - "npm install"

Gruntfile.js

@@ -80,27 +80,11 @@ module.exports = function(grunt) {
             dev: {
                 src: ['front/src/css']
             },
-            coverage: {
-                src: ['.tmp', 'coverage/']
-            },
             build: {
                 src: ['front/build']
             }
         },
         copy: {
-            beforeCoverage: {
-                files: [
-                    {src: ['bin/server.js'], dest: '.tmp/'}
-                ]
-            },
-            coverage: {
-                files: [
-                    {cwd: 'test', src: '**/*', dest: 'coverage/test', expand: true},
-                    {cwd: 'lib/metadata', src: '**/*', dest: 'coverage/lib/metadata', expand: true},
-                    {cwd: 'node_modules/phantomas', src: '**/*', dest: 'coverage/node_modules/phantomas', expand: true},
-                    {cwd: 'lib/tools/phantomas/custom_modules', src: '**/*', dest: 'coverage/lib/tools/phantomas/custom_modules', expand: true}
-                ]
-            },
             build: {
                 files: [
                     {src: ['./front/src/img/favicon.png'], dest: './front/build/img/favicon.png'},
@@ -108,50 +92,18 @@ module.exports = function(grunt) {
                 ]
             }
         },
-        lineremover: {
-            beforeCoverage: {
-                files: {
-                    '.tmp/bin/cli.js': 'bin/cli.js'
-                },
-                options: {
-                    exclusionPattern: /#!\/usr\/bin\/env node/
-                }
-            }
-        },
-        blanket: {
-            coverageApp: {
-                src: ['app/'],
-                dest: 'coverage/app/'
-            },
-            coverageLib: {
-                src: ['lib/'],
-                dest: 'coverage/lib/'
-            },
-            coverageBin: {
-                src: ['.tmp/bin/'],
-                dest: 'coverage/bin/'
-            }
-        },
         mochaTest: {
             test: {
                 options: {
                     reporter: 'spec',
                 },
-                src: ['coverage/test/core/*.js', 'coverage/test/api/*.js']
+                src: ['test/core/*.js', 'test/api/*.js']
             },
             'test-current-work': {
                 options: {
                     reporter: 'spec',
                 },
-                src: ['test/core/imageOptimizerTest.js']
-            },
-            coverage: {
-                options: {
-                    reporter: 'html-cov',
-                    quiet: true,
-                    captureFile: 'coverage/coverage.html'
-                },
-                src: ['coverage/test/core/*.js', 'coverage/test/api/*.js']
+                src: ['test/api/apiTest.js']
             }
         },
         env: {
@@ -182,7 +134,7 @@ module.exports = function(grunt) {
             test: {
                 options: {
                     port: 8387,
-                    server: './coverage/bin/server.js',
+                    server: './bin/server.js',
                     showStack: true
                 }
             },
@@ -243,7 +195,11 @@ module.exports = function(grunt) {
             build: {
                 options: {
                     base: '.tmp',
-                    method: 'append'
+                    method: 'append',
+                    unescape: {
+                        '&lt;': '<',
+                        '&gt;': '>'
+                    }
                 },
                 files: {
                     './front/build/main.html': ['.tmp/views/*.html']
@@ -261,35 +217,11 @@ module.exports = function(grunt) {
         }
     });
 
-
-    // Custom task: copies the test settings.json file to the coverage folder, and checks if there's no missing fields
-    grunt.registerTask('copy-test-server-settings', function() {
-        var mainSettingsFile = './server_config/settings.json';
-        var testSettingsFile = './test/fixtures/settings.json';
-
-        var mainSettings = grunt.file.readJSON(mainSettingsFile);
-        var testSettings = grunt.file.readJSON(testSettingsFile);
-
-        // Recursively compare keys of two objects (not the values)
-        function compareKeys(original, copy, context) {
-            for (var key in original) {
-                if (!copy[key] && copy[key] !== '' && copy[key] !== 0) {
-                    grunt.fail.warn('Settings file ' + testSettingsFile + ' doesn\'t contain key ' + context + '.' + key);
-                }
-                if (original[key] !== null && typeof original[key] === 'object') {
-                    compareKeys(original[key], copy[key], context + '.' + key);
-                }
-            }
-        }
-
-        compareKeys(mainSettings, testSettings, 'settings');
-
-        var outputFile = './coverage/server_config/settings.json';
-        grunt.file.write(outputFile, JSON.stringify(testSettings, null, 4));
-        grunt.log.ok('File ' + outputFile + ' created');
+    // Custom task that sets a variable for tests
+    grunt.registerTask('test-settings', function() {
+        process.env.IS_TEST = true;
     });
 
-
     grunt.registerTask('icons', [
         'webfont:icons',
         'less',
@@ -329,25 +261,18 @@ module.exports = function(grunt) {
     ]);
 
     grunt.registerTask('test', [
+        'test-settings',
         'build',
         'express:testSuite',
-        'clean:coverage',
-        'copy-test-server-settings',
-        'lineremover:beforeCoverage',
-        'copy:beforeCoverage',
-        'blanket',
-        'copy:coverage',
         'express:test',
         'mochaTest:test',
-        'mochaTest:coverage',
         'clean:tmp'
     ]);
 
     grunt.registerTask('test-current-work', [
+        'test-settings',
         'jshint',
         'express:testSuite',
-        'clean:coverage',
-        'copy-test-server-settings',
         'express:test-current-work',
         'mochaTest:test-current-work',
         'clean:tmp'

README.md

@@ -1,6 +1,6 @@
 <img src="./doc/img/logo.png" alt="Yellow Lab Tools"></img>
 
-Analyzes a webpage and detects **performance** or **front-end code quality** issues. Easy to launch, easy to understand, in-depth details.
+Analyzes a webpage and detects **performance** or **front-end code quality** issues. Free, easy to launch, easy to understand, in-depth details.
 
 
 #  
@@ -75,18 +75,15 @@ You can use [ngrok](https://ngrok.com/), a tool that creates a secure tunnel bet
 If your project is not accessible from outside, or if you want to fork and improve the tool, you can build your own instance. The documentation is [here](https://github.com/gmetais/YellowLabTools/wiki/Install-your-private-server).
 
 
-## Help needed!
-There are so many things left to do, **your help would be greatly appreciated**! Please report bugs, ask for evolutions and come code with me.
-
-
-## License
-Please read the [license](LICENSE). Icons are the property of [IcoMoon.io](https://icomoon.io/)
-
-
 ## Author
-Gaël Métais. I'm a webperf freelance based in Paris. If you understand French, you can visit [my website](http://www.gaelmetais.com).
+Gaël Métais. I'm a webperf freelance. Follow me on Twitter [@gaelmetais](https://twitter.com/gaelmetais), I tweet about Web Performances, Front-end and new versions of YellowLabTools!
+
+I can also help your company about Web Performances, visit [my website](https://www.gaelmetais.com).
 
 
 ## Contributors
 - Achraf Ben Younes [achrafbenyounes](https://github.com/achrafbenyounes)
+- [camlafit](https://github.com/camlafit)
+- Vincent L. [magikcypress](https://github.com/magikcypress)
+- Ousama Ben Younes [ousamabenyounes](https://github.com/ousamabenyounes)
 

Vagrantfile

@@ -1,17 +1,24 @@
 Vagrant.configure("2") do |config|
 
-  config.vm.box = "precise64"
-  config.vm.box_url = "http://files.vagrantup.com/precise64.box"
+  config.vm.box = "ubuntu/trusty64"
 
   config.vm.network :private_network, ip: "10.10.10.10"
   config.ssh.forward_agent = true
 
+  # http://foo-o-rama.com/vagrant--stdin-is-not-a-tty--fix.html
+  config.vm.provision "fix-no-tty", type: "shell" do |s|
+    s.privileged = false
+    s.inline = "sudo sed -i '/tty/!s/mesg n/tty -s \\&\\& mesg n/' /root/.profile"
+  end
+
   config.vm.provider :virtualbox do |vb|
     vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]
-    vb.customize ["modifyvm", :id, "--memory", 512]
+    vb.customize ["modifyvm", :id, "--memory", 1024]
     vb.customize ["modifyvm", :id, "--cpus", 2]
   end
 
+  config.vm.synced_folder "./", "/space/YellowLabTools"
+
   config.vm.provision :shell, :path => "server_config/server_install.sh"
 
-end
+end

bin/cli.js

@@ -1,10 +1,11 @@
 #!/usr/bin/env node
 
-var debug = require('debug')('ylt:cli');
-var meow = require('meow');
-var path = require('path');
+var debug       = require('debug')('ylt:cli');
+var meow        = require('meow');
+var path        = require('path');
+var EasyXml     = require('easyxml');
 
-var ylt = require('../lib/index');
+var ylt         = require('../lib/index');
 
 var cli = meow({
     help: [
@@ -14,11 +15,15 @@ var cli = meow({
         'Options:',
         '  --device             Use "phone" or "tablet" to simulate a mobile device (by user-agent and viewport size).',
         '  --screenshot         Will take a screenshot and use this value as the output path. It needs to end with ".png".',
-        '  --js-deep-analysis   When activated, the javascriptExecutionTree will contain sub-requests.',
-        '  --wait-for-selector  Once the page is loaded, Phantomas will wait until the given CSS selector matches some elements.',
+        //'  --wait-for-selector  Once the page is loaded, Phantomas will wait until the given CSS selector matches some elements.',
+        '  --proxy              Sets an HTTP proxy to pass through. Syntax is "host:port".',
         '  --cookie             Adds a cookie on the main domain.',
         '  --auth-user          Basic HTTP authentication username.',
         '  --auth-pass          Basic HTTP authentication password.',
+        '  --block-domain       Disallow requests to given (comma-separated) domains - aka blacklist.',
+        '  --allow-domain       Only allow requests to given (comma-separated) domains - aka whitelist.',
+        '  --no-externals       Block all domains except the main one.',
+        '  --reporter           The output format: "json" or "xml". Default is "json".',
         ''
     ].join('\n'),
     pkg: '../package.json'
@@ -49,17 +54,15 @@ if (screenshot) {
     options.screenshot = cli.flags.screenshot;
 }
 
-// Deep JS analysis option
-if (cli.flags.jsDeepAnalysis === true || cli.flags.jsDeepAnalysis === 'true') {
-    options.jsDeepAnalysis = true;
-}
-
 // Device simulation
 options.device = cli.flags.device || 'desktop';
 
 // Wait for CSS selector
 options.waitForSelector = cli.flags.waitForSelector || null;
 
+// Proxy
+options.proxy = cli.flags.proxy || null;
+
 // Cookie
 options.cookie = cli.flags.cookie || null;
 
@@ -67,6 +70,17 @@ options.cookie = cli.flags.cookie || null;
 options.authUser = cli.flags.authUser || null;
 options.authPass = cli.flags.authPass || null;
 
+// Domain blocking
+options.blockDomain =  cli.flags.blockDomain || null;
+options.allowDomain =  cli.flags.allowDomain || null;
+options.noExternals =  cli.flags.noExternals || null;
+
+// Output format
+if (cli.flags.reporter && cli.flags.reporter !== 'json' && cli.flags.reporter !== 'xml') {
+    console.error('Incorrect parameters: reporter has to be "json" or "xml"');
+    process.exit(1);
+}
+
 
 (function execute(url, options) {
     'use strict';
@@ -76,7 +90,34 @@ options.authPass = cli.flags.authPass || null;
         then(function(data) {
 
             debug('Success');
-            console.log(JSON.stringify(data, null, 2));
+            switch(cli.flags.reporter) {
+                case 'xml':
+                    var serializer = new EasyXml({
+                        manifest: true
+                    });
+
+                    // Remove some heavy parts of the results object
+                    delete data.toolsResults;
+                    delete data.javascriptExecutionTree;
+
+                    var xmlOutput = serializer.render(data);
+
+                    // Remove special chars from XML tags: # [ ]
+                    xmlOutput = xmlOutput.replace(/<([^>]*)#([^>]*)>/g, '<$1>');
+                    xmlOutput = xmlOutput.replace(/<([^>]*)\[([^>]*)>/g, '<$1>');
+                    xmlOutput = xmlOutput.replace(/<([^>]*)\]([^>]*)>/g, '<$1>');
+
+                    // Remove special chars from text content: \n \0
+                    xmlOutput = xmlOutput.replace(/(<[a-zA-Z]*>[^<]*)\n([^<]*<\/[a-zA-Z]*>)/g, '$1$2');
+                    xmlOutput = xmlOutput.replace(/\0/g, '');
+                    xmlOutput = xmlOutput.replace(/\uFFFF/g, '');
+                    xmlOutput = xmlOutput.replace(/\u0002/g, '');
+
+                    console.log(xmlOutput);
+                    break;
+                default:
+                    console.log(JSON.stringify(data, null, 2));
+            }
 
         }).fail(function(err) {
             

bin/server.js

@@ -17,6 +17,12 @@ app.use(cors());
 app.use(wwwRedirectMiddleware);
 app.use(authMiddleware);
 app.use(apiLimitsMiddleware);
+app.use(bodyParser.urlencoded({ extended: true })); // for parsing application/x-www-form-urlencoded
+
+
+// EJS HTML engine
+app.engine('.html', require('ejs').__express);
+app.set('view engine', 'ejs');
 
 
 // Initialize the controllers
@@ -27,6 +33,9 @@ var frontController         = require('../lib/server/controllers/frontController
 // Let's start the server!
 if (!process.env.GRUNTED) {
     var settings = require('../server_config/settings.json');
+
+    app.locals.baseUrl = settings.baseUrl;
+
     server.listen(settings.serverPort, function() {
         console.log('Listening on port %d', server.address().port);
 
@@ -37,4 +46,4 @@ if (!process.env.GRUNTED) {
     });
 }
 
-module.exports = app;
+module.exports = app;

bower.json

@@ -1,13 +0,0 @@
-{
-  "name": "yellowlabtools",
-  "dependencies": {
-    "angular": "1.3.15",
-    "angular-route": "1.3.15",
-    "angular-resource": "1.3.15",
-    "angular-sanitize": "1.3.15",
-    "angular-animate": "1.3.15",
-    "angular-local-storage": "0.2.2",
-    "angular-chart.js": "0.7.1",
-    "Chart.js": "~1.0.2"
-  }
-}

front/src/css/dashboard.css

@@ -109,14 +109,18 @@
 .summary .notations .criteria .table {
   width: 100%;
 }
-.summary .notations .criteria .table > div:hover > div {
+.summary .notations .criteria .table > a {
+  text-decoration: none;
+  color: inherit;
+}
+.summary .notations .criteria .table > a:hover > div {
   background: #EBD8E2;
   cursor: pointer;
 }
-.summary .notations .criteria .table > div:hover > div.info {
+.summary .notations .criteria .table > a:hover > div.info {
   background: #FFF;
 }
-.summary .notations .criteria .table > div:hover > div.info .icon-question {
+.summary .notations .criteria .table > a:hover > div.info .icon-question {
   color: #EBD8E2;
 }
 .summary .notations .criteria .grade {

front/src/css/index.css

@@ -33,7 +33,9 @@
 .settings select {
   font-size: 1em;
 }
-.settings input[type=text] {
+.settings input[type=text],
+.settings input[type=password],
+.settings textarea {
   width: 100%;
   min-width: 4em;
 }
@@ -128,6 +130,7 @@
 .advanced > div > div.label {
   width: 25%;
   white-space: nowrap;
+  vertical-align: middle;
 }
 .advanced .subTable {
   display: table;

front/src/css/rule.css

@@ -29,9 +29,16 @@
   border-radius: 0.5em;
   margin: 0 auto 0.5em;
 }
+.rule h3 {
+  margin-bottom: 0em;
+}
+.rule .okThreshold {
+  font-style: italic;
+  font-size: 0.9em;
+}
 .rule .message {
   width: 80%;
-  margin: 0 auto;
+  margin: 1.5em auto;
 }
 .rule .message p {
   margin: 0.5em;
@@ -94,6 +101,7 @@
   margin: 0.2em 0;
   border-radius: 0.4em;
   z-index: 1;
+  cursor: pointer;
 }
 .offendersTable .offenderButton.opens,
 .value .offenderButton.opens {
@@ -150,6 +158,10 @@
   display: block;
   background: #ffe0cc;
 }
+.offendersTable .smallerOffenders,
+.value .smallerOffenders {
+  font-size: 0.9em;
+}
 .offendersHtml {
   display: inline-block;
 }
@@ -167,7 +179,7 @@
 .checker {
   /* Checkerboard background */
   background-color: #ddd;
-  background-image: linear-gradient(45deg, #aaaaaa 25%, transparent 25%, transparent 75%, #aaaaaa 75%, #aaaaaa), linear-gradient(45deg, #aaaaaa 25%, transparent 25%, transparent 75%, #aaaaaa 75%, #aaaaaa);
+  background-image: linear-gradient(45deg, #AAA 25%, transparent 25%, transparent 75%, #AAA 75%, #AAA), linear-gradient(45deg, #AAA 25%, transparent 25%, transparent 75%, #AAA 75%, #AAA);
   background-size: 1em 1em;
   background-position: 0 0, 0.5em 0.5em;
 }
@@ -207,14 +219,25 @@
   z-index: 2;
   border: 0.2em solid #f1c40f;
 }
+.similarColors {
+  margin: 1em;
+  width: 20em;
+  height: 6em;
+}
+.similarColors > div {
+  display: inline-block;
+  width: 10em;
+  height: 3.5em;
+  padding-top: 2.5em;
+}
 .totalWeightPie {
-  max-width: 39em;
+  max-width: 20em;
   margin: 2em auto 4em;
 }
 .totalWeightPie canvas {
   max-width: inherit;
 }
-.hugeFile {
+.offenderProblem {
   font-weight: bold;
   color: #e74c3c;
 }
@@ -237,8 +260,9 @@
   margin-top: 0.5em;
 }
 .smallPreview {
-  max-height: 1.6em;
-  max-width: 4em;
+  display: block;
+  max-height: 4em;
+  max-width: 8em;
   border: 1px solid #000;
-  margin-top: 0.2em;
+  margin: 1em auto 0.2em;
 }

front/src/css/timeline.css

@@ -62,34 +62,34 @@
   background: #F04DA7;
 }
 .timeline .domComplete.interval {
-  background: #ede3ff;
+  background: #EDE3FF;
 }
 .timeline .domComplete .color {
-  background: #c2a3ff;
+  background: #C2A3FF;
 }
 .timeline .domContentLoadedEnd.interval {
-  background: #d8f0f0;
+  background: #D8F0F0;
 }
 .timeline .domContentLoadedEnd .color {
-  background: #7ecccc;
+  background: #7ECCCC;
 }
 .timeline .domContentLoaded.interval {
-  background: #e0ffd1;
+  background: #E0FFD1;
 }
 .timeline .domContentLoaded .color {
-  background: #a7e846;
+  background: #A7E846;
 }
 .timeline .domInteractive.interval {
-  background: #fffccc;
+  background: #FFFCCC;
 }
 .timeline .domInteractive .color {
-  background: #ffe433;
+  background: #FFE433;
 }
 .timeline .domCreation.interval {
-  background: #ffe0cc;
+  background: #FFE0CC;
 }
 .timeline .domCreation .color {
-  background: #ff6600;
+  background: #FF6600;
 }
 .timeline .tooltip.detailsOverlay {
   position: absolute;
@@ -152,14 +152,16 @@
   width: 100%;
   border-spacing: 0.25em;
 }
-.table > div {
+.table > div,
+.table > a {
   display: table-row;
 }
 .table > .headers > div {
   font-weight: bold;
   padding: 0.5em 1em;
 }
-.table > div > div {
+.table > div > div,
+.table > a > div {
   padding: 0.1em 1em;
   background: #f2f2f2;
   display: table-cell;
@@ -292,19 +294,19 @@
   white-space: nowrap;
 }
 .table > div > .startTime.domComplete {
-  background: #ede3ff;
+  background: #EDE3FF;
 }
 .table > div > .startTime.domContentLoadedEnd {
-  background: #d8f0f0;
+  background: #D8F0F0;
 }
 .table > div > .startTime.domContentLoaded {
-  background: #e0ffd1;
+  background: #E0FFD1;
 }
 .table > div > .startTime.domInteractive {
-  background: #fffccc;
+  background: #FFFCCC;
 }
 .table > div > .startTime.domCreation {
-  background: #ffe0cc;
+  background: #FFE0CC;
 }
 .execution .icon-warning {
   color: #e74c3c;

front/src/js/app.js

@@ -35,7 +35,14 @@ yltApp.run(['$rootScope', '$location', function($rootScope, $location) {
 
     // Google Analytics
     $rootScope.$on('$routeChangeSuccess', function(){
-        ga('send', 'pageview', {'page': $location.path()});
+        if (ga) {
+            ga('send', 'pageview', {'page': $location.path()});
+        }
+    });
+
+    // GitHub star button (asynchronously loaded iframe)
+    window.addEventListener('load', function() {
+        window.document.getElementById('ghbtn').src = 'http://ghbtns.com/github-btn.html?user=gmetais&repo=YellowLabTools&type=star&count=true&size=large';
     });
 }]);
 
@@ -77,3 +84,7 @@ yltApp.config(['$routeProvider', '$locationProvider',
     }
 ]);
 
+// Disable debugging https://docs.angularjs.org/guide/production
+yltApp.config(['$compileProvider', function ($compileProvider) {
+    $compileProvider.debugInfoEnabled(false);
+}]);

front/src/js/controllers/dashboardCtrl.js

@@ -30,10 +30,6 @@ dashboardCtrl.controller('DashboardCtrl', ['$scope', '$rootScope', '$routeParams
         $scope.tweetText = 'I\'ve discovered this cool open-source tool that audits the front-end quality of a web page: ';
     }
 
-    $scope.showRulePage = function(ruleName) {
-        $location.path('/result/' + $scope.runId + '/rule/' + ruleName);
-    };
-
     $scope.testAgain = function() {
         API.relaunchTest($scope.result);
     };

front/src/js/controllers/indexCtrl.js

@@ -1,13 +1,23 @@
 var indexCtrl = angular.module('indexCtrl', []);
 
-indexCtrl.controller('IndexCtrl', ['$scope', 'Settings', 'API', function($scope, Settings, API) {
+indexCtrl.controller('IndexCtrl', ['$scope', '$routeParams', '$location', 'Settings', 'API', function($scope, $routeParams, $location, Settings, API) {
     
     $scope.settings = Settings.getMergedSettings();
 
     $scope.launchTest = function() {
         if ($scope.url) {
+            $location.search('url', null);
+            $location.search('run', null);
             Settings.saveSettings($scope.settings);
             API.launchTest($scope.url, $scope.settings);
         }
     };
+
+    // Auto fill URL field and auto launch test when the good params are set in the URL
+    if ($routeParams.url) {
+        $scope.url = $routeParams.url;
+        if ($routeParams.run === 'true' || $routeParams.run === 1 || $routeParams.run === '1') {
+            $scope.launchTest();
+        }
+    }
 }]);

front/src/js/controllers/ruleCtrl.js

@@ -45,7 +45,85 @@ ruleCtrl.controller('RuleCtrl', ['$scope', '$rootScope', '$routeParams', '$locat
             });
 
             $scope.weightOptions = {
-                tooltipTemplate: '<%=label%>: <%=value%> KB'
+                tooltips: {
+                    callbacks: {
+                        label: function(tooltipItem, data) {
+                            var label = data.labels[tooltipItem.index];
+                            var value = data.datasets[tooltipItem.datasetIndex].data[tooltipItem.index];
+                            return label + ': ' + value + ' KB';
+                        }
+                    }
+                },
+                legend: {
+                    display: true,
+                    position: 'bottom',
+                    labels: {
+                        boxWidth: 12,
+                        fontSize: 14
+                    }
+                }
+            };
+        }
+
+        // Init "Breakpoints" chart
+        if ($scope.policyName === 'cssBreakpoints' && $scope.rule.value > 0) {
+
+            // Seek for the biggest breakpoint
+            var max = 0;
+            $scope.rule.offendersObj.forEach(function(offender) {
+                if (offender.pixels > max) {
+                    max = offender.pixels;
+                }
+            });
+            max = Math.max(max + 100, 1400);
+
+            // We group offenders 10px by 10px
+            var GROUP_SIZE = 20;
+
+            // Generate an empty array of values
+            $scope.breakpointsLabels = [];
+            $scope.breakpointsData = [[]];
+            for (var i = 0; i <= max / GROUP_SIZE; i++) {
+                $scope.breakpointsLabels[i] = '';
+                $scope.breakpointsData[0][i] = 0;
+            }
+
+            // Fill it with results
+            $scope.rule.offendersObj.forEach(function(offender) {
+                var group = Math.floor((offender.pixels + 1) / GROUP_SIZE);
+
+                if ($scope.breakpointsLabels[group] !== '') {
+                    $scope.breakpointsLabels[group] += '/';
+                }
+                $scope.breakpointsLabels[group] += offender.breakpoint;
+
+                $scope.breakpointsData[0][group] += offender.count;
+            });
+
+            $scope.breakpointsSeries = ['Number of CSS rules per breakpoint'];
+            $scope.breakpointsColours = ['#9c4274'];
+            $scope.breakpointsOptions = {
+                scales: {
+                    xAxes: [{
+                        gridLines: {
+                            display:false
+                        }
+                    }],
+                    yAxes: [{
+                        gridLines: {
+                            display:false
+                        }
+                    }]
+                },
+                tooltips: {
+                    enabled: false
+                },
+
+                elements: {
+                    point: {
+                        radius: 0
+                    }
+                }
             };
         }
     }

front/src/js/directives/offendersDirectives.js

@@ -608,7 +608,12 @@
             for (var i = 0 ; i < parsedBacktrace.length ; i++) {
                 html += '<div>';
                 html += '<div>' + (parsedBacktrace[i].fnName || '(anonymous function)') + '</div>';
-                html += '<div class="trace">' + getUrlLink(parsedBacktrace[i].filePath, 40) + ':' + parsedBacktrace[i].line + '</div>';
+                html += '<div class="trace">' + getUrlLink(parsedBacktrace[i].filePath, 40) + '</div>';
+                if (parsedBacktrace[i].column) {
+                    html += '<div>' + parsedBacktrace[i].line + ':' + parsedBacktrace[i].column + '</div>';    
+                } else {
+                    html += '<div>line ' + parsedBacktrace[i].line + '</div>';
+                }
                 html += '</div>';
             }
         }
@@ -630,6 +635,22 @@
                 var fnName = null, fileAndLine;
 
                 var withFnResult = /^([^\s\(]+) \((.+:\d+)\)$/.exec(trace);
+                
+                if (withFnResult === null) {
+                    // Try the PhantomJS 2 format
+                    withFnResult = /^([^\s\(]+) \((.+:\d+:\d+)\)$/.exec(trace);
+                }
+
+                if (withFnResult === null) {
+                    // Yet another PhantomJS 2 format?
+                    withFnResult = /^([^\s\(]+|global code)@(.+:\d+:\d+)$/.exec(trace);
+                }
+
+                if (withFnResult === null) {
+                    // Try the PhantomJS 2 ERROR format
+                    withFnResult = /^([^\s\(]+) (http.+:\d+)$/.exec(trace);
+                }
+
                 if (withFnResult === null) {
                     fileAndLine = trace;
                 } else {
@@ -637,15 +658,26 @@
                     fileAndLine = withFnResult[2];
                 }
 
-                var fileAndLineSplit = /^(.*):(\d+)$/.exec(fileAndLine);
+                // And now the second part
+                var fileAndLineSplit = /^(.*):(\d+):(\d+)$/.exec(fileAndLine);
+
+                if (fileAndLineSplit === null) {
+                    fileAndLineSplit = /^(.*):(\d+)$/.exec(fileAndLine);
+                }
+
                 var filePath = fileAndLineSplit[1];
                 var line = fileAndLineSplit[2];
+                var column = fileAndLineSplit[3];
 
-                out.push({
-                    fnName: fnName,
-                    filePath: filePath,
-                    line: line
-                });
+                // Filter phantomas code
+                if (filePath.indexOf('phantomjs://') === -1) {
+                    out.push({
+                        fnName: fnName,
+                        filePath: filePath,
+                        line: line,
+                        column: column
+                    });
+                }
             });
 
         } catch(e) {
@@ -719,7 +751,7 @@
         }
 
         function onDetailsClick(row) {
-            // Close if it's alreay open
+            // Close if it's already open
             if (row.classList.contains('showDetails')) {
                 closeDetails(row);
                 return;
@@ -886,4 +918,10 @@
         };
     });
 
-})();
+    offendersDirectives.filter('addSpaces', function() {
+        return function(str) {
+            return str.split('').join(' ');
+        };
+    });
+
+})();

front/src/js/models/resultsFactory.js

@@ -1,7 +1,7 @@
 var resultsFactory = angular.module('resultsFactory', ['ngResource']);
 
 resultsFactory.factory('Results', ['$resource', function($resource) {
-    return $resource('/api/results/:runId', {
+    return $resource('api/results/:runId', {
         
     });
 }]);

front/src/js/models/runsFactory.js

@@ -1,7 +1,7 @@
 var runsFactory = angular.module('runsFactory', ['ngResource']);
 
 runsFactory.factory('Runs', ['$resource', function($resource) {
-    return $resource('/api/runs/:runId', {
+    return $resource('api/runs/:runId', {
     
     });
 }]);

front/src/js/services/apiService.js

@@ -9,25 +9,27 @@ apiService.factory('API', ['$location', 'Runs', 'Results', function($location, R
                 url: url,
                 waitForResponse: false,
                 screenshot: true,
-                jsTimeline: true,
                 device: settings.device,
                 waitForSelector: settings.waitForSelector,
+                proxy: settings.proxy,
                 cookie: settings.cookie,
                 authUser: settings.authUser,
                 authPass: settings.authPass,
+                blockDomain: settings.blockDomain,
+                allowedDomains: settings.allowedDomains,
+                noExternals: settings.noExternals
             };
 
-            if (settings.waitForSelector && settings.waitForSelector !== '') {
-                runObject.waitForSelector = settings.waitForSelector;
-            }
-
-            if (settings.cookie && settings.cookie !== '') {
-                runObject.cookie = settings.cookie;
-            }
-
-            if (settings.authUser && settings.authUser !== '' && settings.authPass && settings.authPass !== '') {
-                runObject.authUser = settings.authUser;
-                runObject.authPass = settings.authPass;
+            
+            if (settings.domainsBlackOrWhite === 'black') {
+                runObject.blockDomain = this.parseDomains(settings.domains);
+            } else if (settings.domainsBlackOrWhite === 'white') {
+                var allowedDomains = this.parseDomains(settings.domains);
+                if (allowedDomains.length > 0) {
+                    runObject.allowDomain = allowedDomains;
+                } else {
+                    runObject.noExternals = true;
+                }
             }
 
             Runs.save(runObject, function(data) {
@@ -35,6 +37,8 @@ apiService.factory('API', ['$location', 'Runs', 'Results', function($location, R
             }, function(response) {
                 if (response.status === 429) {
                     alert('Too many requests, you reached the max number of requests allowed in 24h');
+                } else if (response.status === 403) {
+                    alert('This particular query was blocked due to spamming. If you think it\'s an error, please open an issue on GitHub.');
                 } else {
                     alert('An error occured...');
                 }
@@ -43,6 +47,17 @@ apiService.factory('API', ['$location', 'Runs', 'Results', function($location, R
 
         relaunchTest: function(result) {
             this.launchTest(result.params.url, result.params.options);
+        },
+
+        parseDomains: function(textareaContent) {
+            var lines = textareaContent.split('\n');
+            
+            function removeEmptyLines (line) {
+                return line.trim() !== '';
+            }
+
+            // Remove empty lines
+            return lines.filter(removeEmptyLines).join(',');
         }
     };
 

front/src/less/dashboard.less

@@ -108,7 +108,11 @@
 
 .summary .notations .criteria .table {
     width: 100%;
-    > div:hover > div {
+    > a {
+        text-decoration: none;
+        color: inherit;
+    }
+    > a:hover > div {
         background: #EBD8E2;
         cursor: pointer;
         &.info {

front/src/less/index.less

@@ -38,7 +38,7 @@
         font-size: 1em;
     }
 
-    input[type=text] {
+    input[type=text], input[type=password], textarea {
         width: 100%;
         min-width: 4em;
     }
@@ -147,6 +147,7 @@
             &.label {
                 width: 25%;
                 white-space: nowrap;
+                vertical-align: middle;
             }
         }
     }

front/src/less/main.less

@@ -132,8 +132,12 @@ a.linkButton {
     background: #000;
 
     > div {
-        overflow: scroll;
         position: relative;
+        overflow: hidden;
+
+        &:hover {
+            overflow: auto;
+        }
     }
 
     .screenshotImage {
@@ -234,14 +238,6 @@ a.linkButton {
     }
 }
 
-
-.star {
-    font-weight: bold;
-    span {
-        font-size: 1.2em;
-    }
-}
-
 .footer {
     padding: 3em;
     color: #413;
@@ -251,6 +247,9 @@ a.linkButton {
     .version {
         font-size: 0.7em;
     }
+    .github {
+        margin: 1em 0 0 0.5em;
+    }
 }
 
 [class^="icon-"]:before, [class*=" icon-"]:before {

front/src/less/rule.less

@@ -32,9 +32,18 @@
     margin: 0 auto 0.5em;
 }
 
+.rule h3 {
+    margin-bottom: 0em;
+}
+
+.rule .okThreshold {
+    font-style: italic;
+    font-size: 0.9em;
+}
+
 .rule .message {
     width: 80%;
-    margin: 0 auto;
+    margin: 1.5em auto;
     p {
         margin: 0.5em;
     }
@@ -103,6 +112,7 @@
         margin: 0.2em 0;
         border-radius: 0.4em;
         z-index: 1;
+        cursor: pointer;
 
         &.opens {
             padding-right: 0.75em;
@@ -157,6 +167,10 @@
             }
         }
     }
+
+    .smallerOffenders {
+        font-size: 0.9em;
+    }
 }
 
 .offendersHtml {
@@ -225,8 +239,21 @@
     }
 }
 
+.similarColors {
+    margin: 1em;
+    width: 20em;
+    height: 6em;
+
+    > div {
+        display: inline-block;
+        width: 10em;
+        height: 3.5em;
+        padding-top: 2.5em;
+    }
+}
+
 .totalWeightPie {
-    max-width: 39em;
+    max-width: 20em;
     margin: 2em auto 4em;
 
     canvas {
@@ -234,7 +261,7 @@
     }
 }
 
-.hugeFile {
+.offenderProblem {
     font-weight: bold;
     color: #e74c3c;
 }
@@ -262,8 +289,9 @@
 }
 
 .smallPreview {
-    max-height: 1.6em;
-    max-width: 4em;
+    display: block;
+    max-height: 4em;
+    max-width: 8em;
     border: 1px solid #000;
-    margin-top: 0.2em;
+    margin: 1em auto 0.2em;
 }

front/src/less/timeline.less

@@ -173,7 +173,8 @@
     border-spacing: 0.25em;
 }
 
-.table > div {
+.table > div,
+.table > a {
     display: table-row;
 }
 
@@ -182,7 +183,8 @@
     padding: 0.5em 1em;
 }
 
-.table > div > div {
+.table > div > div,
+.table > a > div {
     padding: 0.1em 1em;
     background: #f2f2f2;
     display: table-cell;

front/src/main.html

@@ -2,60 +2,68 @@
 <head>
 	<meta charset="utf-8">
     <title>Yellow Lab Tools</title>
-    <base href="/">
-    <link rel="icon" type="image/png" href="/img/favicon.png">
-    <meta property="og:image" content="/img/logo-large.png" />
+    <base href="<%= baseUrl %>">
+    <link rel="icon" type="image/png" href="img/favicon.png">
+    <meta property="og:image" content="img/logo-large.png" />
     <meta name="description" content="Free online web performance tool. Audit your webpage for performance and front-end quality issues. And it's open-source!" />
 
-    <!-- build:css /css/styles.css-->
-    <link rel="stylesheet" type="text/css" href="/css/main.css">
-    <link rel="stylesheet" type="text/css" href="/css/index.css">
-    <link rel="stylesheet" type="text/css" href="/css/dashboard.css">
-    <link rel="stylesheet" type="text/css" href="/css/queue.css">
-    <link rel="stylesheet" type="text/css" href="/css/rule.css">
-    <link rel="stylesheet" type="text/css" href="/css/screenshot.css">
-    <link rel="stylesheet" type="text/css" href="/css/timeline.css">
-    <link rel="stylesheet" type="text/css" href="/css/about.css">
-    <link rel="stylesheet" type="text/css" href="/bower_components/angular-chart.js/dist/angular-chart.css">
+    <!-- build:css css/styles.css-->
+    <link rel="stylesheet" type="text/css" href="css/main.css">
+    <link rel="stylesheet" type="text/css" href="css/index.css">
+    <link rel="stylesheet" type="text/css" href="css/dashboard.css">
+    <link rel="stylesheet" type="text/css" href="css/queue.css">
+    <link rel="stylesheet" type="text/css" href="css/rule.css">
+    <link rel="stylesheet" type="text/css" href="css/screenshot.css">
+    <link rel="stylesheet" type="text/css" href="css/timeline.css">
+    <link rel="stylesheet" type="text/css" href="css/about.css">
     <!-- endbuild -->
 
-    <!-- build:js /js/all.js -->
-    <script src="/bower_components/angular/angular.min.js"></script>
-    <script src="/bower_components/Chart.js/Chart.min.js"></script>
-    <script src="/bower_components/angular-route/angular-route.min.js"></script>
-    <script src="/bower_components/angular-resource/angular-resource.min.js"></script>
-    <script src="/bower_components/angular-sanitize/angular-sanitize.min.js"></script>
-    <script src="/bower_components/angular-animate/angular-animate.min.js"></script>
-    <script src="/bower_components/angular-local-storage/dist/angular-local-storage.min.js"></script>
-    <script src="/bower_components/angular-chart.js/dist/angular-chart.min.js"></script>
-    <script src="/js/app.js"></script>
-    <script src="/js/controllers/indexCtrl.js"></script>
-    <script src="/js/controllers/dashboardCtrl.js"></script>
-    <script src="/js/controllers/queueCtrl.js"></script>
-    <script src="/js/controllers/ruleCtrl.js"></script>
-    <script src="/js/controllers/screenshotCtrl.js"></script>
-    <script src="/js/controllers/timelineCtrl.js"></script>
-    <script src="/js/models/resultsFactory.js"></script>
-    <script src="/js/models/runsFactory.js"></script>
-    <script src="/js/services/apiService.js"></script>
-    <script src="/js/services/menuService.js"></script>
-    <script src="/js/services/settingsService.js"></script>
+    <link rel="preconnect" href="//www.google-analytics.com">
+    <link rel="dns-prefetch" href="//www.google-analytics.com">
+    <link rel="preconnect" href="//ghbtns.com">
+    <link rel="dns-prefetch" href="//ghbtns.com">
+    <link rel="preconnect" href="https://api.github.com">
+    <link rel="dns-prefetch" href="https://api.github.com">
 
-    <script src="/js/directives/gradeDirective.js"></script>
-    <script src="/js/directives/offendersDirectives.js"></script>
-    <!-- endbuild -->
-<head>
+</head>
 
 <body ng-app="YellowLabTools">
     <div id="header"><h1>Yellow Lab <span class="icon-lab"></span> Tools</h1></div>
     <div id="body" ng-view autoscroll="true"></div>
     <div class="footer">
-        <a href="/about">Learn more</a><br>If you like <b>Yellow Lab Tools</b>, <a href="https://github.com/gmetais/YellowLabTools" target="_blank" class="star">give it a <span>&#9733;</span> on GitHub</a>!<br>
         <span class="version">@@version</span>
+        <br><a href="<%= baseUrl %>about">More about Yellow Lab Tools</a><br>
+        <div class="github"><iframe id="ghbtn" frameborder="0" scrolling="0" width="145px" height="30px"></iframe></div>
     </div>
 
+    <!-- build:js js/all.js -->
+    <script src="node_modules/angular/angular.min.js"></script>
+    <script src="node_modules/chart.js/dist/Chart.min.js"></script>
+    <script src="node_modules/angular-route/angular-route.min.js"></script>
+    <script src="node_modules/angular-resource/angular-resource.min.js"></script>
+    <script src="node_modules/angular-sanitize/angular-sanitize.min.js"></script>
+    <script src="node_modules/angular-animate/angular-animate.min.js"></script>
+    <script src="node_modules/angular-local-storage/dist/angular-local-storage.min.js"></script>
+    <script src="node_modules/angular-chart.js/dist/angular-chart.min.js"></script>
+    <script src="js/app.js"></script>
+    <script src="js/controllers/indexCtrl.js"></script>
+    <script src="js/controllers/dashboardCtrl.js"></script>
+    <script src="js/controllers/queueCtrl.js"></script>
+    <script src="js/controllers/ruleCtrl.js"></script>
+    <script src="js/controllers/screenshotCtrl.js"></script>
+    <script src="js/controllers/timelineCtrl.js"></script>
+    <script src="js/models/resultsFactory.js"></script>
+    <script src="js/models/runsFactory.js"></script>
+    <script src="js/services/apiService.js"></script>
+    <script src="js/services/menuService.js"></script>
+    <script src="js/services/settingsService.js"></script>
+
+    <script src="js/directives/gradeDirective.js"></script>
+    <script src="js/directives/offendersDirectives.js"></script>
+    <!-- endbuild -->
+
     <script>
-        (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)})(window,document,'script','//www.google-analytics.com/analytics.js','ga');if('@@googleAnalyticsId'.indexOf('UA-')===0){ga('create','@@googleAnalyticsId','auto');}
+        if('@@googleAnalyticsId'.indexOf('UA-')===0){(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)})(window,document,'script','https://www.google-analytics.com/analytics.js','ga');ga('create','@@googleAnalyticsId','auto');}
     </script>
 </body>
 </html>

front/src/views/about.html

@@ -5,5 +5,5 @@
 
     <p>By the way, <b>it's free</b> because we are geeks, not businessmen. All we want is <a href="https://github.com/gmetais/YellowLabTools" target="_blank" class="star">a <span>&#9733;</span> on GitHub</a>. It will boost our motivation to add more awesome features!!!</p>
 
-    <p><br><a href="/">Back to index</a></p>
+    <p><br><a href="<%= baseUrl %>">Back to index</a></p>
 </div>

front/src/views/dashboard.html

@@ -1,6 +1,13 @@
 <div ng-include="'views/resultSubHeader.html'"></div>
 <div class="summary board">
     
+    <div ng-if="result.blockedRequests">
+        <b><ng-pluralize count="result.blockedRequests.length" when="{'0': 'No blocked request', 'one': '1 blocked request', 'other': '{} blocked requests'}"></ng-pluralize>:</b>
+        <div ng-repeat="request in result.blockedRequests track by $index">
+            {{request}}
+        </div>
+    </div>
+
     <div class="globalScore" ng-if="globalScore === 0 || globalScore > 0">
         <div>
             <h2>Global score</h2>
@@ -10,7 +17,7 @@
             </div>
         </div>
         <div>
-            <a href="/result/{{result.runId}}/screenshot">
+            <a href="result/{{result.runId}}/screenshot">
                 <div class="screenshotWrapper" ng-class="result.params.options.device || 'desktop'">
                     <div>
                         <img ng-if="result.screenshotUrl" class="screenshotImage" ng-src="{{result.screenshotUrl}}"/>
@@ -28,8 +35,8 @@
             <div class="category">{{category.label}}</div>
             <div class="criteria">
                 <div class="table" title="Click to see details">
-                    <div ng-repeat="ruleName in category.rules" ng-if="result.rules[ruleName]" ng-init="rule = result.rules[ruleName]"
-                         ng-class="{'warning': rule.abnormal}" ng-click="showRulePage(ruleName)">
+                    <a ng-repeat="ruleName in category.rules" ng-if="result.rules[ruleName]" ng-init="rule = result.rules[ruleName]"
+                         ng-class="{'warning': rule.abnormal}" href="result/{{runId}}/rule/{{ruleName}}">
                         <div class="grade">
                             <grade score="rule.score"></grade>
                         </div>
@@ -42,14 +49,14 @@
                             <span ng-if="rule.abnormalityScore <= -300" class="icon-warning"></span>
                         </div>
                         <div class="info"><span class="icon-question"></span></div>
-                    </div>
+                    </a>
                 </div>
             </div>
         </div>
     </div>
 
     <div ng-if="!error && !fromSocialShare" class="apiTip">
-        <b>Did you know? Yellow Lab Tools has a brand new <a href="https://github.com/gmetais/grunt-yellowlabtools" target="_blank">Grunt plugin</a>!
+        <b>Did you know?</b> You can <b>bookmark</b> this result page URL or <b>share it</b>, Yellow Lab Tools keeps test results for years!
     </div>
 
     <div class="tweet" ng-if="!error && !fromSocialShare">
@@ -63,7 +70,7 @@
     <div class="fromShare" ng-if="!error && fromSocialShare">
         <p>Yellow Lab Tools is a free online tool that analyzes performance and front-end quality.</p>
         <a class="linkButton" href="" ng-click="seeFullReport()">See the full report for this page</a>
-        <a class="linkButton" href="/">Test another webpage</a>
+        <a class="linkButton" href="<%= baseUrl %>">Test another webpage</a>
     </div>
 
     <div ng-if="error">

front/src/views/index.html

@@ -2,7 +2,7 @@
 <p class="price">Free and open source!</p>
 
 <form ng-submit="launchTest()" >
-    <input type="{{$root.isTouchDevice? 'url' : 'text'}}" name="url" ng-model="url" placeholder="http://www.mysite.com" class="url" />
+    <input type="text" name="url" ng-model="url" placeholder="http://www.mysite.com" class="url" />
     <input type="submit" value="Launch test" class="launchBtn" ng-class="{disabled: !url}" />
     <div class="settings">
         <div class="device">
@@ -15,11 +15,13 @@
             <span ng-if="!settings.showAdvanced">Advanced settings &nbsp;✚</span>
             <span ng-if="settings.showAdvanced">Hide advanced settings &nbsp;✖</span>
         </a> ]
-        <span class="currentSettings" ng-if="!settings.showAdvanced && (settings.waitForSelector || settings.cookie || settings.authUser || settings.authPass)">
+        <span class="currentSettings" ng-if="!settings.showAdvanced && (settings.waitForSelector || settings.cookie || settings.authUser || settings.authPass || settings.proxy || settings.domains)">
             Currently set:
             <span ng-if="settings.waitForSelector">wait for selector</span>
             <span ng-if="settings.cookie">cookie</span>
             <span ng-if="settings.authUser || settings.authPass">authentication</span>
+            <span ng-if="settings.proxy">proxy</span>
+            <span ng-if="settings.domains">domain blocking</span>
         </span>
         <div class="advanced" ng-show="settings.showAdvanced">
             <!--<div>
@@ -37,12 +39,12 @@
                     Cookie
                     <span class="settingsTooltip">
                         <span class="icon-question"></span>
-                        <div><b>Cookie</b><br><br>Adds a cookie on the main domain.<br><br>Example: "bar=foo;domain=url"</div>
+                        <div><b>Cookie</b><br><br>Adds cookies, separated by a pipe character.<br><br>Example: "bar1=foo1;domain=.domain1.com|bar2=foo2;domain=www.domain2.com"</div>
                     </span>
                 </div>
                 <div><input type="text" name="cookie" ng-model="settings.cookie" /></div>
             </div>
-            <!--<div>
+            <div>
                 <div class="label">
                     Authent
                     <span class="settingsTooltip">
@@ -57,19 +59,36 @@
                     </div>
                     <div>
                         <div><span>password</div>
-                        <div><input type="text" class="authField" name="authPass" ng-model="settings.authPass" /></div>
+                        <div><input type="password" class="authField" name="authPass" ng-model="settings.authPass" /></div>
                     </div>
                 </div>
-            </div>-->
-            <!--<div>
-                <div class="label">Blocked domains</div>
+            </div>
+            <div>
+                <div class="label">
+                    HTTP proxy
+                    <span class="settingsTooltip">
+                        <span class="icon-question"></span>
+                        <div><b>HTTP proxy</b><br><br>Insert here your proxy settings with the format "host:port".<br><br>Example: "192.168.10.0:3333"</div>
+                    </span>
+                </div>
+                <div><input type="text" name="proxy" ng-model="settings.proxy" /></div>
+            </div>
+            <div>
+                <div class="label">
+                    Block domains
+                    <span class="settingsTooltip">
+                        <span class="icon-question"></span>
+                        <div><b>Block some domains</b><br><br>One line per domain or subdomain.<br><br><i><b>Example:</b><br>google-analytics.com<br>ads.yahoo.com<br>ajax.googleapis.com</i><br><br>An empty whitelist will block all domains except the main domain.</div>
+                    </span>
+                </div>
                 <div>
                     <div>
-                        Blacklist / Whitelist
+                        <input type="radio" name="blackOrWhite" ng-model="settings.domainsBlackOrWhite" value="black" />blacklist
+                        <input type="radio" name="blackOrWhite" ng-model="settings.domainsBlackOrWhite" value="white" />whitelist
                     </div>
-                    <textarea name=""></textarea>
+                    <textarea name="domains" ng-model="settings.domains" rows="5"></textarea>
                 </div>
-            </div>-->
+            </div>
         </div>
     </div>
 </form>

front/src/views/queue.html

@@ -6,7 +6,7 @@
         <p class="statusSubMessage">{{status.error}}</p>
         
         <a class="linkButton" href="https://github.com/gmetais/YellowLabTools/issues" target="_blank">Report the issue on GitHub</a>
-        <a class="linkButton" href="/">New test</a>
+        <a class="linkButton" href="<%= baseUrl %>">New test</a>
     </div>
     <div ng-if="status.statusCode == 'awaiting'">
         <div class="status">
@@ -28,7 +28,7 @@
     <div class="status">Error 404 (test not found)</div>
     <p class="statusSubMessage">The server probably just rebooted. We are very sorry about that, please try to launch the test again.</p>
     
-    <a class="linkButton" href="/">New test</a>
+    <a class="linkButton" href="<%= baseUrl %>">New test</a>
 </div>
 <div ng-if="connectionLost == true">
     <div class="status">Connection lost with server</div>

front/src/views/resultSubHeader.html

@@ -1,7 +1,7 @@
 <div>Tested url: &nbsp; <a href="{{result.params.url}}" target="_blank" class="testedUrl">{{result.params.url}}</a></div>
 
 <div class="resultsMenu">
-    <a class="menuItem back" href="/"><div class="icon-arrow-left3"></div><span>New test<span></a>
+    <a class="menuItem back" href="<%= baseUrl %>"><div class="icon-arrow-left3"></div><span>New test<span></a>
     <a class="menuItem restart" href="" ng-click="testAgain()"><div class="icon-loop"></div><span>Test again<span></a>
     <div class="menuItem" ng-class="{active: Menu.getCurrentPage() == 'dashboard'}" ng-click="Menu.changePage('dashboard')"><div class="icon-list"></div><span>Dashboard</span></div>
     <div class="menuItem" ng-class="{active: Menu.getCurrentPage() == 'timeline'}" ng-click="Menu.changePage('timeline')"><div class="icon-bars"></div><span>JS Timeline</span></div>

front/src/views/rule.html

@@ -14,6 +14,14 @@
                 <span ng-if="rule.policy.unit == 'bytes'">{{rule.value | bytes}}</span>
                 <span ng-if="rule.policy.unit != 'bytes'">{{rule.value}}</span>
             </h3>
+            <div class="okThreshold" ng-if="rule.score < 100 && rule.policy.isOkThreshold !== undefined">
+                Have
+                <span ng-if="rule.policy.unit == 'bytes'">{{rule.policy.isOkThreshold | bytes}}</span>
+                <span ng-if="rule.policy.unit != 'bytes'">{{rule.policy.isOkThreshold}}</span>
+                <span ng-if="rule.policy.isOkThreshold > 0 && rule.policy.isOkThreshold < rule.policy.isBadThreshold">or less</span>
+                <span ng-if="rule.policy.isOkThreshold > rule.policy.isBadThreshold">or more</span>
+                to get the 100/100 score.
+            </div>
             <div ng-bind-html="rule.policy.message" class="message"></div>
         </div>
     </div>
@@ -22,7 +30,7 @@
         <p>This rule reached the abnormality threshold, which means there is a real problem you should care about.</p>
     </div>
     <div class="offenders" ng-if="rule.policy.hasOffenders">
-        <h3><ng-pluralize count="rule.offendersObj.count" when="{'0': 'No offenders', 'one': '1 offender', 'other': '{} offenders'}"></ng-pluralize></h3>
+        <h3 ng-if="rule.offendersObj.count >= 0"><ng-pluralize count="rule.offendersObj.count" when="{'0': 'No offenders', 'one': '1 offender', 'other': '{} offenders'}"></ng-pluralize></h3>
 
         <div ng-if="rule.offendersObj.list" class="offendersTable">
             <div ng-repeat="offender in rule.offendersObj.list track by $index">
@@ -88,61 +96,48 @@
                         </span>
                     </div>
 
+                    <div ng-if="policyName === 'cssRules'">
+                        <span ng-if="offender.file === 'inline CSS'">inline CSS</span>
+                        <span ng-if="offender.file !== 'inline CSS'"><url-link url="offender.file" max-length="80"></url-link></span>
+                        : <ng-pluralize count="offender.rules" when="{'0': '0 rule', 'one':'1 rule','other':'{} rules'}"></ng-pluralize>
+                    </div>
+
+                    <div ng-if="policyName === 'similarColors'">
+                        <div class="similarColors checker"><div ng-style="{'background-color': offender.color1, 'color': offender.isDark ? '#FFF' : '#000'}">{{offender.color1}}</div><div ng-style="{'background-color': offender.color2, 'color': offender.isDark ? '#FFF' : '#000'}">{{offender.color2}}</div></div>
+                    </div>
+
                     <div ng-if="policyName === 'cssParsingErrors'">
                         <b>{{offender.error}}</b>
                         <file-and-line file="offender.file" line="offender.line" column="offender.column"></file-and-line>
                         <span ng-if="offender.file">(<a href="http://jigsaw.w3.org/css-validator/validator?profile=css3&usermedium=all&warning=no&uri={{offender.file | encodeURIComponent}}" target="_blank">Check on the W3C validator</a>)</span>
                     </div>
 
-                    <div ng-if="policyName === 'cssComplexSelectors' || policyName === 'cssComplexSelectorsByAttribute' || policyName === 'cssImports' || policyName === 'cssUniversalSelectors' || policyName === 'cssRedundantBodySelectors' || policyName === 'cssRedundantChildNodesSelectors'">
-                        <span ng-if="offender.bolded" ng-bind-html="offender.bolded"></span>
-                        <b ng-if="!offender.bolded">{{offender.css}}</b>
-                        <file-and-line-button file="offender.file" line="offender.line" column="offender.column"></file-and-line-button>
-                    </div>
-
-                    <div ng-if="policyName === 'cssDuplicatedSelectors'">
-                        {{offender.rule}} (<b>x{{offender.occurrences}}</b>)
-                    </div>
-
-                    <div ng-if="policyName === 'cssDuplicatedProperties'">
-                        Property <b>{{offender.property}}</b> duplicated in <b>{{offender.rule}} { }</b>
-                        <file-and-line-button file="offender.file" line="offender.line" column="offender.column"></file-and-line-button>
-                    </div>
-
-                    <div ng-if="policyName === 'cssEmptyRules'">
-                        <b>{{offender.css}} { }</b>
-                        <file-and-line-button file="offender.file" line="offender.line" column="offender.column"></file-and-line-button>
-                    </div>
-
-                    <div ng-if="policyName === 'cssExpressions'">
-                        {{offender.rule}} {{ '{' + offender.property}}: <b>expression(</b>{{offender.expression}}<b>)</b>}
-                        <file-and-line-button file="offender.file" line="offender.line" column="offender.column"></file-and-line-button>
-                    </div>
-
-                    <div ng-if="policyName === 'cssImportants'">
-                        {{offender.rule}} {{ '{' + offender.property}}: {{offender.value}} <b>!important</b>}
-                        <file-and-line-button file="offender.file" line="offender.line" column="offender.column"></file-and-line-button>
-                    </div>
-
-                    <div ng-if="policyName === 'cssOldIEFixes'">
-                        <span ng-if="offender.browser"><b>{{offender.browser}} fix:</b></span>
-                        <span ng-bind-html="offender.bolded"></span>
+                    <div ng-if="policyName === 'cssImports'">
+                        {{offender.css}}
                         <file-and-line-button file="offender.file" line="offender.line" column="offender.column"></file-and-line-button>
                     </div>
 
                     <div ng-if="policyName === 'cssOldPropertyPrefixes'">
-                        {{offender.rule}} {<b>{{offender.property}}</b>: {{offender.value + '}' }}
-                        <br>
-                        <b>{{offender.message}}</b>
-                        <file-and-line-button file="offender.file" line="offender.line" column="offender.column"></file-and-line-button>
+                        <b>{{offender.property}} {{offender.message}}</b>
+                        <div ng-if="offender.rules.length" ng-click="offender.showMore = !offender.showMore" class="offenderButton">
+                            <span ng-if="!offender.showMore">show</span>
+                            <span ng-if="offender.showMore">hide</span>
+                            <ng-pluralize count="offender.rules.length" when="{'one':'1 offender','other':'{} offenders'}"></ng-pluralize>
+                        </div>
+                        <div ng-if="offender.showMore" class="smallerOffenders">
+                            <div ng-repeat="cssRule in offender.rules">
+                                {{cssRule.rule}} {{'{' + offender.property}}: {{cssRule.value + '}' }}
+                                <file-and-line-button file="cssRule.file" line="cssRule.line" column="cssRule.column"></file-and-line-button>
+                            </div>
+                        </div>
                     </div>
 
-                    <div ng-if="policyName === 'lazyLoadableImagesBelowTheFold'">
-                        <url-link url="offender" max-length="100"></url-link>
+                    <div ng-if="policyName === 'lazyLoadableImagesBelowTheFold' || policyName === 'hiddenImages'">
                         <img ng-src="{{offender}}" class="smallPreview checker"></img>
+                        <url-link url="offender" max-length="100"></url-link>
                     </div>
 
-                    <div ng-if="policyName === 'notFound' || policyName === 'closedConnections' || policyName === 'multipleRequests' || policyName === 'cachingDisabled' || policyName === 'cachingNotSpecified'">
+                    <div ng-if="policyName === 'notFound' || policyName === 'emptyRequests' || policyName === 'closedConnections' || policyName === 'multipleRequests' || policyName === 'cachingDisabled' || policyName === 'cachingNotSpecified'">
                         <url-link url="offender" max-length="100"></url-link>
                     </div>
 
@@ -156,15 +151,73 @@
                         (<ng-pluralize count="offender.requests" when="{'one':'1 request','other':'{} requests'}"></ng-pluralize>)
                     </div>
 
-                    <div ng-if="policyName === 'globalVariables' || policyName === 'jQueryVersionsLoaded'">
+                    <div ng-if="policyName === 'globalVariables' || policyName === 'jQueryVersionsLoaded' || policyName === 'synchronousXHR'">
                         {{offender}}
                     </div>
 
+                    <div ng-if="policyName === 'fontsCount'">
+                        <url-link url="offender.url" max-length="70"></url-link>
+                        ({{offender.size | bytes}})
+                    </div>
                 </div>
             </div>
         </div>
 
-        <div ng-if="!rule.offendersObj.list" class="offendersHtml">
+        <div ng-repeat="fileDetails in rule.offendersObj.byFile track by $index">
+            <h3>
+                <ng-pluralize count="fileDetails.count" when="{'one': '1 offender', 'other': '{} offenders'}"></ng-pluralize>
+                in
+                <url-link ng-if="fileDetails.url !== 'Inline CSS'" url="fileDetails.url" max-length="80"></url-link>
+                <span ng-if="fileDetails.url === 'Inline CSS'">inline CSS</span>
+            </h3>
+
+            <div class="offendersTable">
+                <div ng-repeat="offender in fileDetails.offenders track by $index">
+                    <div ng-if="policyName === 'cssComplexSelectors' || policyName === 'cssComplexSelectorsByAttribute' || policyName === 'cssUniversalSelectors' || policyName === 'cssRedundantBodySelectors' || policyName === 'cssRedundantChildNodesSelectors'">
+                        <span ng-if="offender.bolded" ng-bind-html="offender.bolded"></span>
+                        <b ng-if="!offender.bolded">{{offender.css}}</b>
+                        <span ng-if="offender.line !== null && offender.column !== null"> @ {{offender.line}}:{{offender.column}}</span>
+                    </div>
+
+                    <div ng-if="policyName === 'cssMobileFirst'">
+                        <b>{{offender.query}}</b> for <ng-pluralize count="offender.rules" when="{'one':'1 rule','other':'{} rules'}"></ng-pluralize>
+                        <span ng-if="offender.line !== null && offender.column !== null"> @ {{offender.line}}:{{offender.column}}</span>
+                    </div>
+
+                    <div ng-if="policyName === 'cssDuplicatedSelectors'">
+                        {{offender.rule}} (<b>x{{offender.occurrences}}</b>)
+                    </div>
+
+                    <div ng-if="policyName === 'cssDuplicatedProperties'">
+                        Property <b>{{offender.property}}</b> duplicated in <b>{{offender.rule}} { }</b>
+                        <span ng-if="offender.line !== null && offender.column !== null"> @ {{offender.line}}:{{offender.column}}</span>
+                    </div>
+
+                    <div ng-if="policyName === 'cssEmptyRules'">
+                        <b>{{offender.css}} { }</b>
+                        <span ng-if="offender.line !== null && offender.column !== null"> @ {{offender.line}}:{{offender.column}}</span>
+                    </div>
+
+                    <div ng-if="policyName === 'cssExpressions'">
+                        {{offender.rule}} {{ '{' + offender.property}}: <b>expression(</b>{{offender.expression}}<b>)</b>}
+                        <span ng-if="offender.line !== null && offender.column !== null"> @ {{offender.line}}:{{offender.column}}</span>
+                    </div>
+
+                    <div ng-if="policyName === 'cssImportants'">
+                        {{offender.rule}} {{ '{' + offender.property}}: {{offender.value}} <b>!important</b>}
+                        <span ng-if="offender.line !== null && offender.column !== null"> @ {{offender.line}}:{{offender.column}}</span>
+                    </div>
+
+                    <div ng-if="policyName === 'cssOldIEFixes'">
+                        <span ng-if="offender.browser"><b>{{offender.browser}} fix:</b></span>
+                        <span ng-bind-html="offender.bolded"></span>
+                        <span ng-if="offender.line !== null && offender.column !== null"> @ {{offender.line}}:{{offender.column}}</span>
+                    </div>
+                </div>
+            </div>
+        </div>
+
+        <div ng-if="!rule.offendersObj.list && !rule.offendersObj.byFile" class="offendersHtml">
             
             <div ng-if="policyName === 'DOMelementMaxDepth'">
                 <dom-tree tree="rule.offendersObj.tree"></dom-tree>
@@ -195,14 +248,14 @@
     <div ng-if="policyName === 'totalWeight'">
         <h3>Weight by MIME type</h3>
         <div class="totalWeightPie">
-            <canvas class="chart chart-doughnut" data="weightData" labels="weightLabels" options="weightOptions" colours="weightColours" legend="true"></canvas>
+            <canvas class="chart chart-doughnut" chart-data="weightData" chart-labels="weightLabels" chart-options="weightOptions" chart-colors="weightColours"></canvas>
         </div>
         <div ng-repeat="type in weightLabels">
             <h3>{{rule.offendersObj.list.byType[type].totalWeight | bytes}} of {{type}}</h3>
             <div class="offendersTable">
                 <div ng-repeat="request in rule.offendersObj.list.byType[type].requests | orderBy:'-weight'" ng-if="request.weight > 0">
                     <div><url-link url="request.url" max-length="60"></url-link></div>
-                    <div ng-class="{hugeFile: request.weight > 102400}">{{request.weight | bytes}}</div>
+                    <div ng-class="{offenderProblem: request.weight > 102400}">{{request.weight | bytes}}</div>
                 </div>
             </div>
         </div>
@@ -287,6 +340,17 @@
         </div>
     </div>
 
+    <div ng-if="policyName === 'identicalFiles'">
+        <div ng-repeat="offender in rule.offendersObj.list track by $index">
+            <h4>A file of {{offender.weight | bytes}} is loaded {{offender.urls.length}} times:</h4>
+            <div class="offendersTable">
+                <div ng-repeat="url in offender.urls">
+                    <div><url-link url="url" max-length="100"></url-link></div>
+                </div>
+            </div>
+        </div>
+    </div>
+
     <div ng-if="policyName === 'smallRequests'">
         <div ng-repeat="(type, requests) in rule.offendersObj.list.byType">
             <h3><ng-pluralize count="requests.length" when="{'0': 'small ' + type + ' file', 'one': '1 small ' + type + ' file', 'other': '{} small ' + type + ' files'}"></ng-pluralize></h3>
@@ -310,22 +374,106 @@
 
     <div ng-if="policyName === 'DOMaccesses'">
         <h3>{{rule.value}} offenders</h3>
-        Please open the <a href="/result/{{runId}}/timeline">JS timeline</a>
+        Please open the <a href="result/{{runId}}/timeline">JS timeline</a>
     </div>
 
     <div ng-if="policyName === 'queriesWithoutResults'">
         <h3>{{rule.value}} offenders</h3>
-        Please open the <a href="/result/{{runId}}/timeline#filter=queryWithoutResults">JS timeline, filtered by "Queries without results"</a>
+        Please open the <a href="result/{{runId}}/timeline#filter=queryWithoutResults">JS timeline, filtered by "Queries without results"</a>
     </div>
 
     <div ng-if="policyName === 'jQueryCallsOnEmptyObject'">
         <h3>{{rule.value}} offenders</h3>
-        Please open the <a href="/result/{{runId}}/timeline#filter=jQueryCallOnEmptyObject">JS timeline, filtered by "jQuery calls on empty object"</a>
+        Please open the <a href="result/{{runId}}/timeline#filter=jQueryCallOnEmptyObject">JS timeline, filtered by "jQuery calls on empty object"</a>
     </div>
 
     <div ng-if="policyName === 'jQueryNotDelegatedEvents'">
         <h3>{{rule.value}} offenders</h3>
-        Please open the <a href="/result/{{runId}}/timeline#filter=eventNotDelegated">JS timeline, filtered by "Events not delegated"</a>
+        Please open the <a href="result/{{runId}}/timeline#filter=eventNotDelegated">JS timeline, filtered by "Events not delegated"</a>
+    </div>
+
+    <div ng-if="policyName === 'cssBreakpoints'">
+        <div ng-if="rule.value > 0" class="cssBreakpointsGraph">
+            <h3>Breakpoints distribution graph</h3>
+            <canvas class="chart chart-line" chart-data="breakpointsData" chart-labels="breakpointsLabels" chart-options="breakpointsOptions" chart-colors="breakpointsColours" chart-series="breakpointsSeries" width="600" height="250"></canvas>
+            <h3>Breakpoints list</h3>
+            <div class="offendersTable">
+                <div ng-repeat="offender in rule.offendersObj | orderBy:'pixels'">
+                    <div>Breakpoint <b>{{offender.breakpoint}}</b> involves <ng-pluralize count="offender.count" when="{'one': '1 rule', 'other': '{} rules'}"></ng-pluralize></div>
+                </div>
+            </div>
+        </div>
+        <div ng-if="rule.value === 0">
+        No breakpoint
+        </div>
+    </div>
+
+    <div ng-if="policyName === 'heavyFonts'">
+        <div ng-repeat="font in rule.offendersObj.fonts | orderBy:'-weight' track by $index">
+            <h3><url-link url="font.url" max-length="80"></url-link></h3>
+            <div class="offendersTable">
+                <div>
+                    <div>Weight</div>
+                    <div ng-if="font.weight <= 40960">{{font.weight | bytes}}</div>
+                    <div ng-if="font.weight > 40960" class="offenderProblem">{{font.weight | bytes}}</div>
+                </div>
+                <div>
+                    <div>Number of glyphs</div>
+                    <div ng-if="font.numGlyphs <= 500">{{font.numGlyphs}}</div>
+                    <div ng-if="font.numGlyphs > 500" class="offenderProblem">{{font.numGlyphs}} (better &lt; 500)</div>
+                </div>
+                <div>
+                    <div>Average glyph complexity</div>
+                    <div ng-if="font.averageGlyphComplexity <= 35">{{font.averageGlyphComplexity}}</div>
+                    <div ng-if="font.averageGlyphComplexity > 35" class="offenderProblem">{{font.averageGlyphComplexity}} (better &lt; 35)</div>
+                </div>
+            </div>
+        </div>
+    </div>
+
+    <div ng-if="policyName === 'unusedUnicodeRanges'">
+        <div ng-repeat="font in rule.offendersObj.fonts | orderBy:'-compressedWeigth' track by $index">
+            <h3><url-link url="font.url" max-length="60"></url-link> ({{font.weight | bytes}})</h3>
+            <div ng-if="font.isIconFont" class="offendersTable">
+                <div>
+                    <div>
+                        This font seems to be an icon font
+                        <span ng-if="font.numGlyphsInCommonWithPageContent / font.glyphs <= 0.05" class="offenderProblem">but only {{font.numGlyphsInCommonWithPageContent}} of its {{font.glyphs}} glyphs <ng-pluralize count="font.numGlyphsInCommonWithPageContent" when="{'one': 'is', 'other': 'are'}"></ng-pluralize> possibly used!</span>
+                        <span ng-if="font.numGlyphsInCommonWithPageContent / font.glyphs > 0.05">and {{font.numGlyphsInCommonWithPageContent}} of its {{font.glyphs}} glyphs <ng-pluralize count="font.numGlyphsInCommonWithPageContent" when="{'one': 'is', 'other': 'are'}"></ng-pluralize> possibly used.</span>
+                    </div>
+                </div>
+            </div>
+            <div ng-if="!font.isIconFont" class="offendersTable">
+                <div ng-repeat="range in font.unicodeRanges track by $index">
+                    <div><b>{{range.name}}</b></div>
+                    <div ng-if="!range.underused">{{range.numGlyphsInCommonWithPageContent}} of its {{range.charset.length}} glyphs <ng-pluralize count="range.numGlyphsInCommonWithPageContent" when="{'one': 'is', 'other': 'are'}"></ng-pluralize> possibly used</div>
+                    <div ng-if="range.underused" class="offenderProblem">{{range.numGlyphsInCommonWithPageContent}} of its {{range.charset.length}} glyphs are used</div>
+                    <div>
+                        <div class="offenderButton opens">
+                            glyphes list
+                            <div>{{range.charset | addSpaces}}</div>
+                        </div>
+                    </div>
+                </div>
+                <div ng-if="font.ligaturesOrHiddenChars > 0">
+                    <div><b>Ligatures or hidden chars</b></div>
+                    <div ng-class="{offenderProblem: (font.ligaturesOrHiddenChars > 25)}">{{font.ligaturesOrHiddenChars}} glyphs</div>
+                    <div></div>
+                </div>
+            </div>
+        </div>
+    </div>
+
+    <div ng-if="policyName === 'http2'">
+        <h3>Protocols advertised by the server</h3>
+        <div class="offendersTable">
+            <div ng-repeat="protocol in rule.offendersObj.list">
+                <div>{{protocol}}</div>
+            </div>
+            <div ng-if="!rule.offendersObj || rule.offendersObj.count == 0">
+                <div>none</div>
+            </div>
+        </div>
     </div>
 
     <div ng-if="!rule && rule !== null" class="notFound">

lib/metadata/scoreProfileGeneric.json

@@ -15,9 +15,11 @@
                 "totalRequests": 5,
                 "domains": 3,
                 "notFound": 3,
-                "multipleRequests": 2,
+                "identicalFiles": 2,
+                "emptyRequests": 3,
                 "smallRequests": 1,
-                "lazyLoadableImagesBelowTheFold": 2
+                "lazyLoadableImagesBelowTheFold": 2,
+                "hiddenImages": 1
             }
         },
         "domComplexity": {
@@ -49,6 +51,7 @@
             "policies": {
                 "jsErrors": 1,
                 "documentWriteCalls": 2,
+                "synchronousXHR": 5,
                 "consoleMessages": 0.5,
                 "globalVariables": 0.5
             }
@@ -58,7 +61,6 @@
             "policies": {
                 "jQueryVersion": 2,
                 "jQueryVersionsLoaded": 2,
-                "jQueryFunctionsUsed": 1,
                 "jQueryCallsOnEmptyObject": 1,
                 "jQueryNotDelegatedEvents": 1
             }
@@ -68,8 +70,10 @@
             "policies": {
                 "cssRules": 2,
                 "cssComplexSelectors": 2,
-                "cssComplexSelectorsByAttribute": 1.5,
-                "cssColors": 0.5
+                "cssColors": 0.5,
+                "similarColors": 0.5,
+                "cssBreakpoints": 0.5,
+                "cssMobileFirst": 1
             }
         },
         "badCSS": {
@@ -84,14 +88,22 @@
                 "cssImportants": 3,
                 "cssOldIEFixes": 1,
                 "cssOldPropertyPrefixes": 1,
-                "cssUniversalSelectors": 1,
                 "cssRedundantBodySelectors": 1,
                 "cssRedundantChildNodesSelectors": 1
             }
         },
+        "fonts": {
+            "label": "Web fonts",
+            "policies": {
+                "fontsCount": 1,
+                "heavyFonts": 1,
+                "unusedUnicodeRanges": 1
+            }
+        },
         "serverConfig": {
             "label": "Server config",
             "policies": {
+                "http2": 2,
                 "closedConnections": 2,
                 "cachingNotSpecified": 1,
                 "cachingDisabled": 1,
@@ -105,11 +117,12 @@
         "domComplexity": 2,
         "domManipulations": 2,
         "scroll": 1,
-        "badJavascript": 1,
+        "badJavascript": 2,
         "jQuery": 1,
         "cssSyntaxError": 1,
         "cssComplexity": 1,
         "badCSS": 1,
+        "fonts": 1,
         "serverConfig": 1
     }
 }

lib/offendersHelpers.js

@@ -113,14 +113,17 @@ var OffendersHelpers = function() {
         if (traceArray) {
             var results = [];
             var parts = null;
+            var obj;
 
             for (var i=0 ; i<traceArray.length ; i++) {
-                parts = /^(([\w$]+) )?\(?([^ ]+):(\d+)\)?$/.exec(traceArray[i]);
+                // Handle the new PhantomJS 2.x syntax
+                parts = /^(([\w$]+)@)?([^ ]+):(\d+):(\d+)$/.exec(traceArray[i]);
 
                 if (parts) {
-                    var obj = {
+                    obj = {
                         file: parts[3],
-                        line: parseInt(parts[4], 10)
+                        line: parseInt(parts[4], 10),
+                        column: parseInt(parts[5], 10)
                     };
 
                     if (parts[2]) {
@@ -128,8 +131,26 @@ var OffendersHelpers = function() {
                     }
 
                     results.push(obj);
+
                 } else {
-                    return null;
+                    // Old syntax
+                    parts = /^(([\w$]+) )?\(?([^ ]+):(\d+)\)?$/.exec(traceArray[i]);
+
+                    if (parts) {
+                        obj = {
+                            file: parts[3],
+                            line: parseInt(parts[4], 10)
+                        };
+
+                        if (parts[2]) {
+                            obj.functionName = parts[2];
+                        }
+
+                        results.push(obj);
+
+                    } else {
+                        return null;
+                    }
                 }
             }
             return results;
@@ -151,7 +172,10 @@ var OffendersHelpers = function() {
     };
 
     this.cssOffenderPattern = function(offender) {
-        var parts = /^(.*) (?:<([^ \(]*)>|\[inline CSS\]) @ (\d+):(\d+)$/.exec(offender);
+        // Remove any line breaks
+        offender = offender.replace(/(\r\n|\n|\r)/gm, '');
+
+        var parts = /^(.*) (?:<([^ \(]*)>|\[inline CSS\]) ?@ ?(\d+):(\d+)$/.exec(offender);
         
         if (!parts) {
             return {
@@ -182,6 +206,34 @@ var OffendersHelpers = function() {
         }
     };
 
+    this.orderByFile = function(offenders) {
+        var byFileObj = {};
+
+        offenders.forEach(function(offender) {
+            var file = offender.file || 'Inline CSS';
+            delete offender.file;
+
+            if (!byFileObj[file]) {
+                byFileObj[file] = {
+                    url: file,
+                    count: 0,
+                    offenders: []
+                };
+            }
+
+            byFileObj[file].count ++;
+            byFileObj[file].offenders.push(offender);
+        });
+
+        // Transform object into array
+        var byFileArray = [];
+        for (var file in byFileObj) {
+            byFileArray.push(byFileObj[file]);
+        }
+
+        return {byFile: byFileArray};
+    };
+
 };
 
 module.exports = new OffendersHelpers();

lib/runner.js

@@ -3,7 +3,10 @@ var debug                   = require('debug')('ylt:runner');
 
 var phantomasWrapper        = require('./tools/phantomas/phantomasWrapper');
 var jsExecutionTransformer  = require('./tools/jsExecutionTransformer');
-var weightChecker           = require('./tools/weightChecker/weightChecker');
+var colorDiff               = require('./tools/colorDiff');
+var mediaQueriesChecker     = require('./tools/mediaQueriesChecker');
+var isHttp2                 = require('./tools/isHttp2');
+var redownload              = require('./tools/redownload/redownload');
 var rulesChecker            = require('./rulesChecker');
 var scoreCalculator         = require('./scoreCalculator');
 
@@ -28,9 +31,20 @@ var Runner = function(params) {
         // Treat the JS Execution Tree from offenders
         data = jsExecutionTransformer.transform(data);
 
-        // Redownload every file
-        return weightChecker.recheckAllFiles(data);
+        // Compare colors
+        data = colorDiff.compareAllColors(data);
 
+        // Check media queries
+        data = mediaQueriesChecker.analyzeMediaQueries(data);
+
+        // Redownload every file
+        return redownload.recheckAllFiles(data);
+
+    })
+
+    .then(function(data) {
+        // Check if HTTP2
+        return isHttp2.check(data);
     })
 
     .then(function(data) {
@@ -52,15 +66,14 @@ var Runner = function(params) {
         delete data.toolsResults.phantomas.metrics.scrollExecutionTree;
         delete data.toolsResults.phantomas.offenders.scrollExecutionTree;
 
-        return data;
 
-    })
+        if (data.toolsResults.phantomas.offenders.blockedRequests) {
+            data.blockedRequests = data.toolsResults.phantomas.offenders.blockedRequests;
+        }
 
-    .then(function(data) {
 
         // Finished!
         deferred.resolve(data);
-
     })
 
     .fail(function(err) {

lib/screenshotHandler.js

@@ -1,5 +1,5 @@
 var debug       = require('debug')('ylt:screenshotHandler');
-var lwip        = require('lwip');
+var Jimp        = require('jimp');
 var tmp         = require('temporary');
 var Q           = require('q');
 var fs          = require('fs');
@@ -57,7 +57,7 @@ var screenshotHandler = function() {
     this.openImage = function(imagePath) {
         var deferred = Q.defer();
 
-        lwip.open(imagePath, function(err, image){
+        Jimp.read(imagePath, function(err, image){
             if (err) {
                 debug('Could not open imagePath %s', imagePath);
                 debug(err);
@@ -76,7 +76,7 @@ var screenshotHandler = function() {
     this.resizeImage = function(image, newWidth) {
         var deferred = Q.defer();
 
-        var currentWidth = image.width();
+        var currentWidth = image.bitmap.width;
         var ratio = newWidth / currentWidth;
 
         image.scale(ratio, function(err, image){
@@ -101,7 +101,7 @@ var screenshotHandler = function() {
         var deferred = Q.defer();
 
         // Create a canvas with the same dimensions as your image:
-        lwip.create(image.width(), image.height(), 'white', function(err, canvas){
+        new Jimp(image.bitmap.width, image.bitmap.height, 0xFFFFFF, function(err, canvas){
             if (err) {
                 debug('Could not create a white canvas');
                 debug(err);
@@ -109,7 +109,7 @@ var screenshotHandler = function() {
                 deferred.reject(err);
             } else {
                 // Paste original image on top of the canvas
-                canvas.paste(0, 0, image, function(err, image){
+                canvas.composite(image, 0, 0, function(err, image){
                     if (err) {
                         debug('Could not paste image on the white canvas');
                         debug(err);
@@ -131,7 +131,7 @@ var screenshotHandler = function() {
     this.toBuffer = function(image) {
         var deferred = Q.defer();
 
-        image.toBuffer('jpg', {quality: 90}, function(err, buffer){
+        image.quality(90).getBuffer(Jimp.MIME_JPEG, function(err, buffer){
             if (err) {
                 debug('Could not save image to buffer');
                 debug(err);
@@ -143,7 +143,7 @@ var screenshotHandler = function() {
             }
         });
 
-        return deferred.promise;        
+        return deferred.promise;
     };
 
 
@@ -167,4 +167,4 @@ var screenshotHandler = function() {
     };
 };
 
-module.exports = new screenshotHandler();
+module.exports = new screenshotHandler();

lib/server/controllers/apiController.js

@@ -7,6 +7,7 @@ var RunsQueue           = require('../datastores/runsQueue');
 var RunsDatastore       = require('../datastores/runsDatastore');
 var ResultsDatastore    = require('../datastores/resultsDatastore');
 
+var serverSettings      = (process.env.IS_TEST) ? require('../../../test/fixtures/settings.json') : require('../../../server_config/settings.json');
 
 var ApiController = function(app) {
     'use strict';
@@ -15,8 +16,6 @@ var ApiController = function(app) {
     var runsDatastore = new RunsDatastore();
     var resultsDatastore = new ResultsDatastore();
 
-
-
     // Create a new run
     app.post('/api/runs', function(req, res) {
 
@@ -25,20 +24,30 @@ var ApiController = function(app) {
             req.body.url = 'http://' + req.body.url;
         }
 
+        // Block requests to unwanted websites (=spam)
+        if (req.body.url && isBlocked(req.body.url)) {
+            console.error('Test blocked for URL: %s', req.body.url);
+            res.status(403).send('Forbidden');
+            return;
+        }
+
         // Grab the test parameters and generate a random run ID
         var run = {
             runId: (Date.now()*1000 + Math.round(Math.random()*1000)).toString(36),
             params: {
                 url: req.body.url,
-                waitForResponse: req.body.waitForResponse !== false && req.body.waitForResponse !== 'false' && req.body.waitForResponse !== 0,
+                waitForResponse: req.body.waitForResponse === true || req.body.waitForResponse === 'true' || req.body.waitForResponse === 1,
                 partialResult: req.body.partialResult || null,
                 screenshot: req.body.screenshot || false,
-                jsTimeline: req.body.jsTimeline || false,
                 device: req.body.device || 'desktop',
+                proxy: req.body.proxy || null,
                 waitForSelector: req.body.waitForSelector || null,
                 cookie: req.body.cookie || null,
                 authUser: req.body.authUser || null,
-                authPass: req.body.authPass || null
+                authPass: req.body.authPass || null,
+                blockDomain: req.body.blockDomain || null,
+                allowDomain: req.body.allowDomain || null,
+                noExternals: req.body.noExternals || false
             }
         };
 
@@ -52,7 +61,6 @@ var ApiController = function(app) {
         debug('Adding test %s to the queue', run.runId);
         var queuePromise = queue.push(run.runId);
 
-
         // Save the run to the datastore
         runsDatastore.add(run, queuePromise.startingPosition);
 
@@ -67,21 +75,26 @@ var ApiController = function(app) {
 
             runsDatastore.updatePosition(run.runId, 0);
 
-            debug('Launching test %s on %s', run.runId, run.params.url);
+            console.log('Launching test ' + run.runId + ' on ' + run.params.url);
 
             var runOptions = {
                 screenshot: run.params.screenshot ? screenshot.getTmpFilePath() : false,
-                jsDeepAnalysis: run.params.jsTimeline,
                 device: run.params.device,
+                proxy: run.params.proxy,
                 waitForSelector: run.params.waitForSelector,
                 cookie: run.params.cookie,
                 authUser: run.params.authUser,
-                authPass: run.params.authPass
+                authPass: run.params.authPass,
+                blockDomain: run.params.blockDomain,
+                allowDomain: run.params.allowDomain,
+                noExternals: run.params.noExternals,
+                phantomasEngine: serverSettings.phantomasEngine
             };
 
             return ylt(run.params.url, runOptions);
 
         })
+
         // Phantomas completed, let's save the screenshot if any
         .then(function(data) {
 
@@ -95,7 +108,7 @@ var ApiController = function(app) {
             if (run.params.screenshot) {
                 
                 // Replace the empty promise created earlier with Q.resolve()
-                screenshotPromise = screenshot.toThumbnail(400)
+                screenshotPromise = screenshot.toThumbnail(serverSettings.screenshotWidth || 400)
                 
                     // Read screenshot
                     .then(function(screenshotBuffer) {
@@ -105,7 +118,7 @@ var ApiController = function(app) {
                             data.screenshotBuffer = screenshotBuffer;
 
                             // Official path to get the image
-                            data.screenshotUrl = '/api/results/' + data.runId + '/screenshot.jpg';
+                            data.screenshotUrl = 'api/results/' + data.runId + '/screenshot.jpg';
                         }
 
                     })
@@ -128,13 +141,7 @@ var ApiController = function(app) {
                     // Remove uneeded temp screenshot path
                     delete data.params.options.screenshot;
 
-                    // Empty javascriptExecutionTree if not needed
-                    if (!run.params.jsTimeline) {
-                        data.javascriptExecutionTree = {};
-                        data.scrollExecutionTree = {};
-                    }
-
-                    // Remove tools results if not needed
+                    // Here we can remove tools results if not needed
 
                     return resultsDatastore.saveResult(data);
                 })
@@ -197,7 +204,7 @@ var ApiController = function(app) {
 
         // The user doesn't want to wait for the response, sending the run ID only
         if (!run.params.waitForResponse) {
-            console.log('Sending response without waiting.');
+            debug('Sending response without waiting.');
             res.setHeader('Content-Type', 'application/json');
             res.send(JSON.stringify({runId: run.runId}));
         }
@@ -329,6 +336,15 @@ var ApiController = function(app) {
             });
     });
 
+    function isBlocked(url) {
+        if (!serverSettings.blockedUrls) {
+            return false;
+        }
+
+        return serverSettings.blockedUrls.some(function(blockedUrl) {
+            return (url.indexOf(blockedUrl) === 0);
+        });
+    }
 };
 
-module.exports = ApiController;
+module.exports = ApiController;

lib/server/controllers/frontController.js

@@ -6,21 +6,33 @@ var FrontController = function(app) {
 
     var cacheDuration = 365 * 24 * 60 * 60 * 1000; // One year
     var assetsPath = (app.get('env') === 'development') ? '../../../front/src' : '../../../front/build';
-    
+
+    // Routes templating    
     var routes = ['/', '/about', '/result/:runId', '/result/:runId/timeline', '/result/:runId/screenshot', '/result/:runId/rule/:policy', '/queue/:runId'];
+
     routes.forEach(function(route) {
         app.get(route, function(req, res) {
             res.setHeader('Cache-Control', 'public, max-age=20');
-            res.sendFile(path.join(__dirname, assetsPath, 'main.html'));
+            res.render(path.join(__dirname, assetsPath, 'main.html'), {
+                baseUrl: app.locals.baseUrl || '/'
+            });
+        });
+    });
+
+    // Views templating
+    app.get('/views/:viewName', function(req, res) {
+        res.setHeader('Cache-Control', 'public, max-age=' + cacheDuration);
+        res.render(path.join(__dirname, assetsPath, 'views/' + req.params.viewName), {
+            baseUrl: app.locals.baseUrl || '/'
         });
     });
     
+    // Static assets
     app.use('/css', express.static(path.join(__dirname, assetsPath, 'css'), { maxAge: cacheDuration }));
     app.use('/fonts', express.static(path.join(__dirname, assetsPath, 'fonts'), { maxAge: cacheDuration }));
     app.use('/img', express.static(path.join(__dirname, assetsPath, 'img'), { maxAge: cacheDuration }));
     app.use('/js', express.static(path.join(__dirname, assetsPath, 'js'), { maxAge: cacheDuration }));
-    app.use('/views', express.static(path.join(__dirname, assetsPath, 'views'), { maxAge: cacheDuration }));
-    app.use('/bower_components', express.static(path.join(__dirname, '../../../bower_components'), { maxAge: cacheDuration }));
+    app.use('/node_modules', express.static(path.join(__dirname, '../../../node_modules'), { maxAge: cacheDuration }));
 };
 
 module.exports = FrontController;

lib/server/middlewares/apiLimitsMiddleware.js

@@ -1,6 +1,6 @@
-var config      = require('../../../server_config/settings.json');
+var config = (process.env.IS_TEST) ? require('../../../test/fixtures/settings.json') : require('../../../server_config/settings.json');
 
-var debug       = require('debug')('apiLimitsMiddleware');
+var debug = require('debug')('apiLimitsMiddleware');
 
 
 var apiLimitsMiddleware = function(req, res, next) {

lib/server/middlewares/authMiddleware.js

@@ -1,6 +1,6 @@
-var config      = require('../../../server_config/settings.json');
+var config = (process.env.IS_TEST) ? require('../../../test/fixtures/settings.json') : require('../../../server_config/settings.json');
 
-var debug       = require('debug')('authMiddleware');
+var debug = require('debug')('authMiddleware');
 
 
 var authMiddleware = function(req, res, next) {

lib/tools/colorDiff.js

@@ -0,0 +1,92 @@
+var debug   = require('debug')('ylt:colorDiff');
+var diff    = require('color-diff');
+var parse   = require('parse-color');
+
+
+var colorDiff = function() {
+
+    var COLOR_DIFF_THRESHOLD = 0.9;
+
+    this.compareAllColors = function(data) {
+        debug('Starting to compare all colors...');
+
+        var offenders = data.toolsResults.phantomas.offenders.cssColors;
+        var colors = (offenders) ? this.parseAllColors(offenders) : [];
+        var result = [];
+
+        // Compare colors to each other
+        for (var i = 0; i < colors.length - 1 ; i++) {
+            for (var j = i + 1; j < colors.length; j++) {
+                debug('Comparing color %s with color %s', colors[i].original, colors[j].original);
+                if (this.compareTwoColors(colors[i], colors[j])) {
+                    debug('Colors are similar!');
+                    result.push({
+                        color1: colors[i].original,
+                        color2: colors[j].original,
+                        isDark: (colors[i].Lab.L < 60)
+                    });
+                }
+            }
+        }
+
+        data.toolsResults.colorDiff = {
+            metrics: {
+                similarColors: result.length
+            },
+            offenders: {
+                similarColors: result
+            }
+        };
+
+        return data;
+    };
+
+    this.parseAllColors = function(offenders) {
+        var parsedOffenders = offenders.map(this.parseOffender);
+        var deduplicatedColors = {};
+
+        parsedOffenders.forEach(function(color) {
+            if (color !== null) {
+                deduplicatedColors[color] = color;
+            }
+        });
+
+        return Object.keys(deduplicatedColors).map(this.parseColor);
+    };
+
+    this.parseOffender = function(offender) {
+        var regexResult = /^(.*) \(\d+ times\)$/.exec(offender);
+        return regexResult ? regexResult[1] : null;
+    };
+
+    this.parseColor = function(color) {
+        var colorArray = parse(color).rgba;
+
+        var obj = {
+            R: colorArray[0],
+            G: colorArray[1],
+            B: colorArray[2],
+            A: colorArray[3]
+        };
+
+        obj.Lab = diff.rgb_to_lab(obj);
+        obj.original = color;
+
+        return obj;
+    };
+
+    this.compareTwoColors = function(color1, color2) {
+        if (color1.A !== color2.A) {
+            debug('Comparison not possible, because the alpha channel is different');
+            return false;
+        }
+
+        var delta = diff.diff(color1.Lab, color2.Lab);
+        debug('Delta is %d', delta);
+        
+        return delta <= COLOR_DIFF_THRESHOLD;
+    };
+
+};
+
+module.exports = new colorDiff();

lib/tools/isHttp2.js

@@ -0,0 +1,105 @@
+var debug   = require('debug')('ylt:isHttp2');
+var url     = require('url');
+var Q       = require('q');
+var http2   = require('is-http2');
+
+var isHttp2 = function() {
+    'use strict';
+
+    this.check = function(data) {
+        debug('Starting to check for HTTP2 support...');
+
+        return this.checkHttp2(data)
+
+            .then(function(result) {
+
+                if (result.isHttp) {
+                    debug('The website is not even in HTTPS');
+
+                    data.toolsResults.http2 = {
+                        metrics: {
+                            http2: false
+                        }
+                    };
+
+                } else if (result.isHttp2) {
+                    debug('HTTP/2 (or SPDY) is supported');
+
+                    data.toolsResults.http2 = {
+                        metrics: {
+                            http2: true
+                        }
+                    };
+                
+                } else {
+                    debug('HTTP/2 is not supported');
+                    
+                    data.toolsResults.http2 = {
+                        metrics: {
+                            http2: false
+                        }
+                    };
+                }
+
+                // Add the supported protocols as offenders
+                if (result.supportedProtocols) {
+                    debug('Supported protocols: ' + result.supportedProtocols.join(' '));
+                    data.toolsResults.http2.offenders = {
+                        http2: result.supportedProtocols
+                    };
+                }
+
+                debug('End of HTTP2 support check');
+
+                return data;
+            })
+
+            .fail(function() {
+                return data;
+            });
+    };
+
+    this.getParsedUrl = function(data) {
+        return url.parse(data.toolsResults.phantomas.url);
+    };
+
+    this.getProtocol = function(data) {
+        return this.getParsedUrl(data).protocol;
+    };
+
+    this.getDomain = function(data) {
+        return this.getParsedUrl(data).hostname;
+    };
+
+    this.checkHttp2 = function(data) {
+        var deferred = Q.defer();
+
+        // Check if it's HTTPS first
+        if (this.getProtocol(data) === 'http:') {
+            
+            deferred.resolve({
+                isHttp: true
+            });
+
+        } else {
+
+            // To make is-http2 work, you need to have openssl in a version greater than 1.0.0 installed and available in your $path.
+            http2(this.getDomain(data), {includeSpdy: true})
+
+                .then(function(result) {
+                    deferred.resolve(result);
+                })
+                
+                .catch(function(error) {
+                    debug('Error while checking HTTP2 support:');
+                    debug(error);
+                    deferred.reject('Error while checking for HTTP2 support');
+                });
+
+        }
+
+        return deferred.promise;
+    };
+};
+
+module.exports = new isHttp2();

lib/tools/jsExecutionTransformer.js

@@ -25,11 +25,8 @@ var jsExecutionTransformer = function() {
         var hasjQuery = (data.toolsResults.phantomas.metrics.jQueryVersionsLoaded > 0);
         if (hasjQuery) {
             metrics.jQueryCalls = 0;
-            metrics.jQueryFunctionsUsed = 0;
             metrics.jQueryCallsOnEmptyObject = 0;
             metrics.jQueryNotDelegatedEvents = 0;
-
-            offenders.jQueryFunctionsUsed = [];
         }
 
         try {
@@ -102,43 +99,32 @@ var jsExecutionTransformer = function() {
                     metrics.DOMaccesses += countTreeLeafs(node);
                 });
 
-                // Count the number of different jQuery functions called
-                if (hasjQuery) {
-
-                    jQueryFunctionsCollection.sort().forEach(function(fnName, cnt) {
-                        if (fnName === 'jQuery - find') {
-                            fnName = 'jQuery - $';
-                        }
-                        metrics.jQueryFunctionsUsed ++;
-                        offenders.jQueryFunctionsUsed.push({
-                            functionName: fnName.substring(9),
-                            count: cnt
-                        });
-                    });
-                }
-
             }
             debug('JS execution transformation complete');
 
 
-            debug('Starting scroll execution transformation');
-            offenders.DOMaccessesOnScroll = JSON.parse(data.toolsResults.phantomas.offenders.scrollExecutionTree[0]);
-            if (offenders.DOMaccessesOnScroll.children) {
-                offenders.DOMaccessesOnScroll.children.forEach(function(node) {
-                    
-                    // Mark a event flag
-                    if (['documentScroll', 'windowScroll', 'window.onscroll'].indexOf(node.data.type) >= 0) {
-                        node.windowPerformance = true;
-                    }
+            if (data.toolsResults.phantomas.offenders.scrollExecutionTree) {
+                debug('Starting scroll execution transformation');
+                offenders.DOMaccessesOnScroll = JSON.parse(data.toolsResults.phantomas.offenders.scrollExecutionTree[0]);
+                if (offenders.DOMaccessesOnScroll.children) {
+                    offenders.DOMaccessesOnScroll.children.forEach(function(node) {
+                        
+                        // Mark a event flag
+                        if (['documentScroll', 'windowScroll', 'window.onscroll'].indexOf(node.data.type) >= 0) {
+                            node.windowPerformance = true;
+                        }
 
-                    // Transform domPaths into objects
-                    changeListOfDomPaths(node);
-                    
-                    // Count the number of DOM accesses, by counting the tree leafs
-                    metrics.DOMaccessesOnScroll += countTreeLeafs(node);
-                });
+                        // Transform domPaths into objects
+                        changeListOfDomPaths(node);
+                        
+                        // Count the number of DOM accesses, by counting the tree leafs
+                        metrics.DOMaccessesOnScroll += countTreeLeafs(node);
+                    });
+                }
+                debug('Scroll execution transformation complete');
+            } else {
+                debug('Could not parse scrollExecutionTree');
             }
-            debug('Scroll execution transformation complete');
 
         } catch(err) {
             throw err;

lib/tools/mediaQueriesChecker.js

@@ -0,0 +1,168 @@
+var debug   = require('debug')('ylt:mediaQueriesChecker');
+var parseMediaQuery = require('css-mq-parser');
+var offendersHelpers = require('../offendersHelpers');
+
+
+var mediaQueriesChecker = function() {
+    'use strict';
+
+    var MOBILE_MIN_BREAKPOINT = 200;
+    var MOBILE_MAX_BREAKPOINT = 300;
+
+    this.analyzeMediaQueries = function(data) {
+        debug('Starting to check all media queries...');
+
+        var offenders = data.toolsResults.phantomas.offenders.cssMediaQueries;
+        var mediaQueries = (offenders) ? this.parseAllMediaQueries(offenders) : [];
+        
+        var notMobileFirstCount = 0;
+        var notMobileFirstOffenders = [];
+
+        var breakpointsOffenders = {};
+
+        for (var i = 0; i < mediaQueries.length; i++) {
+            var item = mediaQueries[i];
+
+            if (!item) {
+                continue;
+            }
+
+            if (item.isForMobile) {
+                notMobileFirstCount += item.mediaQuery.rules;
+                notMobileFirstOffenders.push(item.mediaQuery);
+            }
+
+            for (var j = 0; j < item.breakpoints.length; j++) {
+                var breakpointString = item.breakpoints[j].string;
+                if (!breakpointsOffenders[breakpointString]) {
+                    breakpointsOffenders[breakpointString] = {
+                        count: 1,
+                        pixels: item.breakpoints[j].pixels
+                    };
+                } else {
+                    breakpointsOffenders[breakpointString].count += 1;
+                }
+            }
+        }
+
+        data.toolsResults.mediaQueriesChecker = {
+            metrics: {
+                cssMobileFirst: notMobileFirstCount,
+                cssBreakpoints: Object.keys(breakpointsOffenders).length
+            },
+            offenders: {
+                cssMobileFirst: notMobileFirstOffenders,
+                cssBreakpoints: breakpointsOffenders
+            }
+        };
+
+        debug('End of media queries check');
+
+        return data;
+    };
+
+    this.parseAllMediaQueries = function(offenders) {
+        return offenders.map(this.parseOneMediaQuery);
+    };
+
+    this.parseOneMediaQuery = function(offender) {
+        var splittedOffender = offendersHelpers.cssOffenderPattern(offender);
+        var parts = /^@media (.*) \((\d+ rules)\)$/.exec(splittedOffender.css);
+
+        if (!parts) {
+            debug('Failed to parse media query ' + offender);
+            return false;
+        }
+
+        var rulesCount = parseInt(parts[2], 10);
+        var query = parts[1];
+
+        var isForMobile = false;
+        var breakpoints = [];
+
+        try {
+
+            var ast = parseMediaQuery(query);
+
+            var min = 0;
+            var max = Infinity;
+            var pixels;
+
+            ast.forEach(function(astItem) {
+                astItem.expressions.forEach(function(expression) {
+                    if (expression.feature === 'width' || expression.feature === 'device-width') {
+                        if (astItem.inverse === false) {
+                            if (expression.modifier === 'max') {
+                                pixels = toPixels(expression.value);
+                                max = Math.min(max, pixels);
+                                breakpoints.push({
+                                    string: expression.value,
+                                    pixels: pixels
+                                });
+                            } else if (expression.modifier === 'min') {
+                                pixels = toPixels(expression.value);
+                                min = Math.max(min, pixels);
+                                breakpoints.push({
+                                    string: expression.value,
+                                    pixels: pixels
+                                });
+                            }
+                        } else if (astItem.inverse === true) {
+                            if (expression.modifier === 'max') {
+                                pixels = toPixels(expression.value);
+                                min = Math.max(min, pixels);
+                                breakpoints.push({
+                                    string: expression.value,
+                                    pixels: pixels
+                                });
+                            } else if (expression.modifier === 'min') {
+                                pixels = toPixels(expression.value);
+                                max = Math.min(max, pixels);
+                                breakpoints.push({
+                                    string: expression.value,
+                                    pixels: pixels
+                                });
+                            }
+                        }
+                    }
+                });
+            });
+
+            isForMobile = (min <= MOBILE_MIN_BREAKPOINT && max >= MOBILE_MAX_BREAKPOINT && max !== Infinity);
+
+        } catch(error) {
+            debug('Failed to parse media query ' + offender);
+        }
+
+        return {
+            mediaQuery: {
+                query: query,
+                rules: rulesCount,
+                file: splittedOffender.file,
+                line: splittedOffender.line,
+                column: splittedOffender.column
+            },
+            isForMobile: isForMobile,
+            breakpoints: breakpoints
+        };
+    };
+
+    // Parses a size in em, pt (or px) and returns it in px
+    function toPixels(size) {
+        var splittedSize = /^([\d\.]+)(.*)/.exec(size);
+        var value = parseFloat(splittedSize[1]);
+        var unit = splittedSize[2];
+
+        if (unit === 'em') {
+            return value * 16;
+        }
+
+        if (unit === 'pt') {
+            return value / 12 * 16;
+        }
+
+        return value;
+    }
+};
+
+module.exports = new mediaQueriesChecker();

lib/tools/phantomas/custom_modules/core/scopeYLT/scopeYLT.js

@@ -18,8 +18,8 @@ exports.module = function(phantomas) {
         responseEndTime = Date.now();
     });
 
-    phantomas.once('init', function() {
-        phantomas.evaluate(function(responseEndTime, deepAnalysis) {
+    phantomas.on('init', function() {
+        phantomas.evaluate(function(responseEndTime) {
             (function(phantomas) {
     
                 // Overwritting phantomas spy function
@@ -49,7 +49,7 @@ exports.module = function(phantomas) {
                             var err;
                             
                             // Before
-                            if (enabled) {
+                            if (enabled && callbackBefore) {
                                 callbackBefore.apply(this, arguments);
                             }
 
@@ -71,7 +71,8 @@ exports.module = function(phantomas) {
 
                                 // After
                                 if (enabled && callbackAfter) {
-                                    callbackAfter.call(this, result, arguments);
+                                    var args = Array.prototype.slice.call(arguments);
+                                    callbackAfter.apply(this, [result].concat(args));
                                 }
 
                                 if (err) {
@@ -106,48 +107,85 @@ exports.module = function(phantomas) {
                     var currentContext = root;
                     var depth = 0;
 
-                    if (deepAnalysis) {
-                        phantomas.log('Entering deep Javascript analysis mode');
-                    }
 
-                    // Add a child but don't enter his context
+                    // Add a child but don't enter its context
                     function pushContext(data) {
-                        if (depth === 0 || deepAnalysis) {
+                        
+                        // Some data is not needed on subchildren
+                        if (depth === 0) {
                             data.timestamp = Date.now() - responseEndTime;
                             data.loadingStep = phantomas.currentStep || '';
-                            currentContext.addChild(data);
                         }
+
+                        // Some data is not needed on subchildren
+                        if (depth > 0) {
+                            if (data.backtrace) {
+                                delete data.backtrace;
+                            }
+                            if (data.resultsNumber) {
+                                delete data.resultsNumber;
+                            }
+                        }
+
+                        currentContext.addChild(data);
                     }
                     
-                    // Add a child to the current context and enter his context
+                    // Add a child to the current context and enter its context
                     function enterContext(data) {
-                        if (depth === 0 || deepAnalysis) {
+                        
+                        // Some data is not needed on subchildren
+                        if (depth === 0) {
                             data.timestamp = Date.now() - responseEndTime;
                             data.loadingStep = phantomas.currentStep || '';
-                            currentContext = currentContext.addChild(data);
                         }
+
+                        // Some data is not needed on subchildren
+                        if (depth > 0) {
+                            if (data.backtrace) {
+                                delete data.backtrace;
+                            }
+                            if (data.resultsNumber) {
+                                delete data.resultsNumber;
+                            }
+                        }
+
+                        currentContext = currentContext.addChild(data);
+
                         depth ++;
                     }
                     
                     // Save given data in the current context and jump change current context to its parent
                     function leaveContext(moreData) {
-                        if (depth === 1 || deepAnalysis) {
+                        
+                        // Some data is not needed on subchildren
+                        if (depth === 1) {
                             currentContext.data.time = Date.now() - currentContext.data.timestamp - responseEndTime;
+                        }
 
-                            // Merge previous data with moreData (ovewrites if exists)
-                            if (moreData) {
-                                for (var key in moreData) {
-                                    currentContext.data[key] = moreData[key];
-                                }
+                        // Some data is not needed on subchildren
+                        if (depth > 1) {
+                            if (moreData && moreData.backtrace) {
+                                delete moreData.backtrace;
                             }
-
-                            var parent = currentContext.parent;
-                            if (parent === null) {
-                                console.error('Error: trying to close root context in ContextTree');
-                            } else {
-                                currentContext = parent;
+                            if (moreData && moreData.resultsNumber) {
+                                delete moreData.resultsNumber;
                             }
                         }
+
+                        // Merge previous data with moreData (ovewrites if exists)
+                        if (moreData) {
+                            for (var key in moreData) {
+                                currentContext.data[key] = moreData[key];
+                            }
+                        }
+
+                        var parent = currentContext.parent;
+                        if (parent === null) {
+                            console.error('Error: trying to close root context in ContextTree');
+                        } else {
+                            currentContext = parent;
+                        }
+
                         depth --;
                     }
                     
@@ -161,7 +199,6 @@ exports.module = function(phantomas) {
                         if (root !== currentContext) {
                             return null;
                         }
-                        var current = currentContext;
 
                         function recusiveRead(node) {
                             if (node.children.length === 0) {
@@ -209,6 +246,6 @@ exports.module = function(phantomas) {
                 })();
 
             })(window.__phantomas);
-        }, responseEndTime, phantomas.getParam('js-deep-analysis'));
+        }, responseEndTime);
     });
 };

lib/tools/phantomas/custom_modules/modules/ajaxReqYLT/ajaxReqYLT.js

@@ -0,0 +1,30 @@
+/**
+ * Analyzes AJAX requests
+ */
+/* global window: true */
+
+exports.version = '0.2.a';
+
+exports.module = function(phantomas) {
+    'use strict';
+
+    phantomas.setMetric('ajaxRequests'); // @desc number of AJAX requests
+    phantomas.setMetric('synchronousXHR'); // @desc number of synchronous 
+
+    phantomas.on('init', function() {
+        phantomas.evaluate(function() {
+            (function(phantomas) {
+                phantomas.spy(window.XMLHttpRequest.prototype, 'open', null, function(result, method, url, async) {
+                    phantomas.incrMetric('ajaxRequests');
+                    phantomas.addOffender('ajaxRequests', '<%s> [%s]', url, method);
+
+                    if (async === false) {
+                        phantomas.incrMetric('synchronousXHR');
+                        phantomas.addOffender('synchronousXHR', url);
+                        phantomas.log('ajaxRequests: synchronous XMLHttpRequest call to <%s>', url);
+                    }
+                }, true);
+            })(window.__phantomas);
+        });
+    });
+};

lib/tools/phantomas/custom_modules/modules/charactersCount/charactersCount.js

@@ -0,0 +1,71 @@
+/**
+ * Lists the number of different characters
+ */
+/* global document: true, Node: true, window: true */
+
+exports.version = '1.0';
+
+exports.module = function(phantomas) {
+    'use strict';
+
+    //phantomas.setMetric('differentCharacters'); // @desc the number of different characters in the body @offenders
+
+    phantomas.on('report', function() {
+        phantomas.log("charactersCount: starting to analyze characters on page...");
+
+        phantomas.evaluate(function() {
+            (function(phantomas) {
+
+                phantomas.spyEnabled(false, 'analyzing characters on page');
+
+                function getLetterCount(arr){
+                    return arr.reduce(function(prev, next){
+                        prev[next] = 1;
+                        return prev;
+                    }, {});
+                }
+
+                if (document.body && document.body.textContent) {
+                    var allText = '';
+
+                    // Traverse all DOM Tree to read text
+                    var runner = new phantomas.nodeRunner();
+                    runner.walk(document.body, function(node, depth) {
+                        switch (node.nodeType) {
+
+                            // Grabing text nodes
+                            case Node.TEXT_NODE:
+                                if (node.parentNode.tagName !== 'SCRIPT' && node.parentNode.tagName !== 'STYLE') {
+                                    allText += node.nodeValue;
+                                }
+                                break;
+
+                            // Grabing CSS content properties
+                            case Node.ELEMENT_NODE:
+                                if (node.tagName !== 'SCRIPT' && node.tagName !== 'STYLE') {
+                                    allText += window.getComputedStyle(node).getPropertyValue('content');
+                                    allText += window.getComputedStyle(node, ':before').getPropertyValue('content');
+                                    allText += window.getComputedStyle(node, ':after').getPropertyValue('content');
+                                }
+                                break;
+                        }
+                    });
+
+                    // Reduce all text found in page into a string of unique chars
+                    var charactersList = getLetterCount(allText.split(''));
+                    var charsetString = Object.keys(charactersList).sort().join('');
+                    
+                    // Remove blank characters
+                    charsetString = charsetString.replace(/\s/g, '');
+
+                    phantomas.setMetric('differentCharacters', charsetString.length);
+                    phantomas.addOffender('differentCharacters', charsetString);
+                }
+
+                phantomas.spyEnabled(true);
+            }(window.__phantomas));
+        });
+
+        phantomas.log("charactersCount: analyzing characters done.");
+    });
+};

lib/tools/phantomas/custom_modules/modules/domHiddenYLT/domHiddenYLT.js

@@ -3,21 +3,23 @@
  */
 /* global document: true, Node: true, window: true */
 
-exports.version = '0.1.a';
+exports.version = '1.0.a';
 
 exports.module = function(phantomas) {
     'use strict';
 
     // total length of HTML of hidden elements (i.e. display: none)
     phantomas.setMetric('hiddenContentSize'); // @desc the size of content of hidden elements on the page (with CSS display: none) @offenders
+    phantomas.setMetric('hiddenImages'); // @desc number of hidden images that can be lazy-loaded @offenders
 
     // HTML size
     phantomas.on('report', function() {
         phantomas.evaluate(function() {
             (function(phantomas) {
-                phantomas.spyEnabled(false, 'checking the hiddenContentSize');
+                var runner = new phantomas.nodeRunner(),
+                    lazyLoadableImages = {};
 
-                var runner = new phantomas.nodeRunner();
+                phantomas.spyEnabled(false, 'analyzing hidden content');
 
                 runner.walk(document.body, function(node, depth) {
                     switch (node.nodeType) {
@@ -36,6 +38,31 @@ exports.module = function(phantomas) {
                                     }
                                 }
 
+                                // count hidden images that can be lazy loaded (issue #524)
+                                var images = [];
+                                if (node.tagName === 'IMG') {
+                                    images = [node];
+                                } else if (typeof node.querySelectorAll === 'function') {
+                                    images = node.querySelectorAll('img') || [];
+                                }
+
+                                for (var i = 0, len = images.length; i < len; i++) {
+                                    var src = images[i].src,
+                                        path;
+
+                                    if (src === '' || src.indexOf('data:image') === 0) continue;
+
+                                    if (images[i].width === 1 && images[i].height === 1) continue;
+
+                                    if (!lazyLoadableImages[src]) {
+                                        path = phantomas.getDOMPath(images[i]);
+
+                                        lazyLoadableImages[src] = {
+                                            path: path
+                                        };
+                                    }
+                                }
+
                                 // don't run for child nodes as they're hidden as well
                                 return false;
                             }
@@ -43,6 +70,15 @@ exports.module = function(phantomas) {
                     }
                 });
 
+                Object.keys(lazyLoadableImages).forEach(function(img) {
+                    var entry = lazyLoadableImages[img];
+
+                    phantomas.incrMetric('hiddenImages');
+                    phantomas.addOffender('hiddenImages', img);
+
+                    phantomas.log('hiddenImages: <%s> image (%s) is hidden and can be lazy-loaded', img, entry.path);
+                });
+
                 phantomas.spyEnabled(true);
             }(window.__phantomas));
         });

lib/tools/phantomas/custom_modules/modules/domQYLT/domQYLT.js

@@ -19,7 +19,7 @@ exports.module = function(phantomas) {
     phantomas.setMetric('DOMqueriesAvoidable'); // @desc number of repeated uses of a duplicated query
 
     // fake native DOM functions
-    phantomas.once('init', function() {
+    phantomas.on('init', function() {
         phantomas.evaluate(function() {
             (function(phantomas) {
                 function querySpy(type, query, fnName, context, hasNoResults) {
@@ -39,9 +39,9 @@ exports.module = function(phantomas) {
                     });
 
                 }, function(result, args) {
-                    var id = args[0];
+                    var id = args ? '#' + args[0] : undefined;
 
-                    querySpy('id', '#' + id, 'getElementById', '#document', (result === null));
+                    querySpy('id', id, 'getElementById', '#document', (result === null));
 
                     var moreData = {
                         resultsNumber : (result === null) ? 0 : 1
@@ -74,10 +74,10 @@ exports.module = function(phantomas) {
                 function selectorClassNameAfter(result, args) {
                     /*jshint validthis: true */
 
-                    var className = args[0];
+                    var className = args ? '.' + args[0] : undefined;
                     var context = phantomas.getDOMPath(this);
 
-                    querySpy('class', '.' + className, 'getElementsByClassName', context, (result.length === 0));
+                    querySpy('class', className, 'getElementsByClassName', context, (result.length === 0));
                     
                     var moreData = {
                         resultsNumber : (result && result.length > 0) ? result.length : 0
@@ -113,10 +113,10 @@ exports.module = function(phantomas) {
                 function selectorTagNameSpyAfter(result, args) {
                     /*jshint validthis: true */
                     
-                    var tagName = args[0];
+                    var tagName = args ? args[0].toLowerCase() : undefined;
                     var context = phantomas.getDOMPath(this);
 
-                    querySpy('tag name', tagName.toLowerCase(), 'getElementsByTagName', context, (result.length === 0));
+                    querySpy('tag name', tagName, 'getElementsByTagName', context, (result.length === 0));
                     
                     var moreData = {
                         resultsNumber : (result && result.length > 0) ? result.length : 0
@@ -156,7 +156,7 @@ exports.module = function(phantomas) {
                 function selectorQuerySpyAfter(result, args) {
                     /*jshint validthis: true */
 
-                    var selector = args[0];
+                    var selector = args ? args[0] : undefined;
                     var context = phantomas.getDOMPath(this);
 
                     querySpy('selector', selector, 'querySelectorAll', context, (!result || result.length === 0));
@@ -189,7 +189,7 @@ exports.module = function(phantomas) {
                 function selectorAllQuerySpryAfter(result, args) {
                     /*jshint validthis: true */
 
-                    var selector = args[0];
+                    var selector = args ? args[0] : undefined;
                     var context = phantomas.getDOMPath(this);
 
                     querySpy('selector', selector, 'querySelectorAll', context, (!result || result.length === 0));

lib/tools/phantomas/custom_modules/modules/eventYLT/eventYLT.js

@@ -12,7 +12,7 @@ exports.module = function(phantomas) {
     phantomas.setMetric('eventsDispatched'); // @desc number of EventTarget.dispatchEvent calls
     phantomas.setMetric('eventsScrollBound'); // @desc number of scroll event bounds
 
-    phantomas.once('init', function() {
+    phantomas.on('init', function() {
         phantomas.evaluate(function() {
             (function(phantomas) {
                 // spy calls to EventTarget.addEventListener

lib/tools/phantomas/custom_modules/modules/jQYLT/jQYLT.js

@@ -138,10 +138,9 @@ exports.module = function(phantomas) {
     jQueryFunctions = jQueryFunctions.concat(jQueryTraversalFunctions);
 
     // spy calls to jQuery functions
-    phantomas.once('init', function() {
+    phantomas.on('init', function() {
         phantomas.evaluate(function(jQueryFunctions, jQueryTraversalFunctions) {
             (function(phantomas) {
-                var jQuery;
                 var oldJQuery;
 
                 phantomas.spyGlobalVar('jQuery', function(jQuery) {
@@ -371,14 +370,6 @@ exports.module = function(phantomas) {
     });
 
 
-    // store the last resource that was received
-    // try to report where given jQuery version was loaded from
-    phantomas.on('recv', function(entry) {
-        if (entry.isJS) {
-            lastUrl = entry.url;
-        }
-    });
-
     phantomas.on('jQueryLoaded', function(version) {
         phantomas.log('jQuery: loaded v' + version);
         phantomas.setMetric('jQueryVersion', version);
@@ -386,7 +377,5 @@ exports.module = function(phantomas) {
         // report multiple jQuery "instances" (issue #435)
         phantomas.incrMetric('jQueryVersionsLoaded');
         phantomas.addOffender('jQueryVersionsLoaded', 'v%s', version);
-
-        phantomas.log('jQuery: v%s (probably loaded from <%s>)', version, lastUrl);
     });
 };

lib/tools/phantomas/custom_modules/modules/javaScriptBottleYLT/javaScriptBottleYLT.js

@@ -23,7 +23,7 @@ exports.module = function(phantomas) {
         phantomas.log('javaScriptBottlenecks: to spy calls to eval() run phantomas with --spy-eval option');
     }
 
-    phantomas.once('init', function() {
+    phantomas.on('init', function() {
         phantomas.evaluate(function(spyEval) {
             (function(phantomas) {
                 function report(msg, caller, backtrace, metric) {

lib/tools/phantomas/custom_modules/modules/lazyLoadableYLT/lazyLoadableYLT.js

@@ -0,0 +1,81 @@
+/**
+ * Analyzes images and detects which one can be lazy-loaded (are below the fold)
+ *
+ * @see https://github.com/macbre/phantomas/issues/494
+ */
+/* global document: true, window: true */
+
+exports.version = '1.0.a';
+
+exports.module = function(phantomas) {
+    'use strict';
+    
+    phantomas.setMetric('lazyLoadableImagesBelowTheFold'); // @desc number of images displayed below the fold that can be lazy-loaded
+
+    phantomas.on('report', function() {
+        phantomas.log('lazyLoadableImages: analyzing which images can be lazy-loaded...');
+
+        phantomas.evaluate(function() {
+            (function(phantomas) {
+                phantomas.spyEnabled(false, 'analyzing which images can be lazy-loaded');
+
+                var images = document.body.getElementsByTagName('img'),
+                    i,
+                    len = images.length,
+                    offset,
+                    path,
+                    processedImages = {},
+                    src,
+                    viewportHeight = window.innerHeight,
+                    // Add an offset of 100px under the height of the screen
+                    LAZYLOAD_OFFSET = 100;
+
+                phantomas.log('lazyLoadableImages: %d image(s) found, assuming %dpx offset to be the fold', len, viewportHeight);
+
+                for (i = 0; i < len; i++) {
+                    // @see https://developer.mozilla.org/en-US/docs/Web/API/Element/getBoundingClientRect
+                    offset = images[i].getBoundingClientRect().top;
+                    src = images[i].src;
+
+                    // ignore base64-encoded images
+                    if (src === '' || /^data:/.test(src)) {
+                        continue;
+                    }
+
+                    path = phantomas.getDOMPath(images[i]);
+
+                    // get the most top position for a given image (deduplicate by src)
+                    if (typeof processedImages[src] === 'undefined') {
+                        processedImages[src] = {
+                            offset: offset,
+                            path: path
+                        };
+                    }
+
+                    // maybe there's the same image loaded above the fold?
+                    if (offset < processedImages[src].offset) {
+                        processedImages[src] = {
+                            offset: offset,
+                            path: path
+                        };
+                    }
+                }
+
+                phantomas.log('lazyLoadableImages: checking %d unique image(s)', Object.keys(processedImages).length);
+
+                Object.keys(processedImages).forEach(function(src) {
+                    var img = processedImages[src];
+
+                    if (img.offset > viewportHeight + LAZYLOAD_OFFSET) {
+                        phantomas.log('lazyLoadableImages: <%s> image (%s) is below the fold (at %dpx)', src, img.path, img.offset);
+
+                        phantomas.incrMetric('lazyLoadableImagesBelowTheFold');
+                        phantomas.addOffender('lazyLoadableImagesBelowTheFold', src);
+                    }
+                });
+
+                phantomas.spyEnabled(true);
+            })(window.__phantomas);
+        });
+    });
+};

lib/tools/phantomas/phantomasWrapper.js

@@ -17,13 +17,12 @@ var PhantomasWrapper = function() {
         var deferred = Q.defer();
         var task = data.params;
 
-
         var options = {
             
             // Cusomizable options
-            'timeout': task.options.timeout || 45,
-            'js-deep-analysis': task.options.jsDeepAnalysis || false,
-            'user-agent': (task.options.device === 'desktop') ? 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.110 Safari/537.36' : null,
+            'engine': task.options.phantomasEngine || 'webkit',
+            'timeout': task.options.timeout || 30,
+            'user-agent': (task.options.device === 'desktop') ? 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) YLT Chrome/27.0.1453.110 Safari/537.36' : null,
             'tablet': (task.options.device === 'tablet'),
             'phone': (task.options.device === 'phone'),
             'screenshot': task.options.screenshot || false,
@@ -31,23 +30,28 @@ var PhantomasWrapper = function() {
             'cookie': task.options.cookie,
             'auth-user': task.options.authUser,
             'auth-pass': task.options.authPass,
+            'block-domain': task.options.blockDomain,
+            'allow-domain': task.options.allowDomain,
+            'no-externals': task.options.noExternals,
 
             // Mandatory
             'reporter': 'json:pretty',
             'analyze-css': true,
+            'ignore-ssl-errors': true,
             'skip-modules': [
-                'blockDomains', // not needed
-                'domHiddenContent', // overriden
+                'ajaxRequests', // overridden
+                'domHiddenContent', // overridden
                 'domMutations', // not compatible with webkit
-                'domQueries', // overriden
+                'domQueries', // overridden
                 'events', // overridden
                 'filmStrip', // not needed
                 'har', // not needed for the moment
                 'javaScriptBottlenecks', // needs to be launched after custom module scopeYLT
                 'jQuery', // overridden
                 'jserrors', // overridden
+                'lazyLoadableImages', //overridden
                 'pageSource', // not needed
-                'windowPerformance' // overriden
+                'windowPerformance' // overridden
             ].join(','),
             'include-dirs': [
                 path.join(__dirname, 'custom_modules/core'),
@@ -55,6 +59,12 @@ var PhantomasWrapper = function() {
             ].join(',')
         };
 
+        // Proxy option can't be set to null or undefined...
+        // this is why it's set now and not in the object above
+        if (task.options.proxy) {
+            options.proxy = task.options.proxy;
+        }
+
         // Output the command line for debugging purpose
         debug('If you want to reproduce the phantomas task only, copy the following command line:');
         var optionsString = '';
@@ -112,9 +122,14 @@ var PhantomasWrapper = function() {
 
         // It's time to launch the test!!!
         var triesNumber = 2;
+        var currentTry = 0;
 
         async.retry(triesNumber, function(cb) {
+
+            currentTry ++;
+
             var process = phantomas(task.url, options, function(err, json, results) {
+                var errorCode = err ? parseInt(err.message, 10) : null;
                 
                 if (isKilled) {
                     debug('Process was killed, too late Phantomas, sorry...');
@@ -122,28 +137,33 @@ var PhantomasWrapper = function() {
                 }
 
 
-                debug('Returning from Phantomas with error %s', err);
+                debug('Returning from Phantomas with error %s', errorCode);
 
                 // Adding some YellowLabTools errors here
                 if (json && json.metrics && (!json.metrics.javascriptExecutionTree || !json.offenders.javascriptExecutionTree)) {
-                    err = 1001;
+                    errorCode = 1001;
                 }
 
-                if (!err && (!json || !json.metrics)) {
-                    err = 1002;
+                if (!errorCode && (!json || !json.metrics)) {
+                    errorCode = 1002;
                 }
 
                 // Don't cancel test if it is a timeout and we've got some results
-                if (err === 252 && json) {
+                if (errorCode === 252 && json) {
                     debug('Timeout after ' + options.timeout + ' seconds. But it\'s not a problem, the test is valid.');
-                    err = null;
+                    errorCode = null;
                 }
 
-                if (err) {
-                    debug('Attempt failed. Error code ' + err);
+                if (errorCode) {
+                    debug('Attempt failed. Error code ' + errorCode);
                 }
 
-                cb(err, json);
+                cb(errorCode, json);
+            
+            }).fail(function() {
+                // This function is useless, but the failing promise needs to be handled,
+                // otherwise the module meow writes in the console in case of a timeout (error code 252).
+                debug('Failing promise handled');
             });
             
             phantomasPid = process.pid;

lib/tools/redownload/contentTypeChecker.js

@@ -0,0 +1,188 @@
+var debug   = require('debug')('ylt:contentTypeChecker');
+var Q       = require('q');
+var isJpg   = require('is-jpg');
+var isPng   = require('is-png');
+var isSvg   = require('is-svg');
+var isGif   = require('is-gif');
+var isWoff  = require('is-woff');
+var isWoff2 = require('is-woff2');
+var isOtf   = require('is-otf');
+var isTtf   = require('is-ttf');
+var isEot   = require('is-eot');
+
+var ContentTypeChecker = function() {
+
+    function checkContentType(entry) {
+        var deferred = Q.defer();
+
+        debug('Entering contentTypeChecker');
+        
+        // Ignore very small files as they are generally tracking pixels
+        if (entry.weightCheck && entry.weightCheck.bodyBuffer && entry.weightCheck.bodySize > 100) {
+            var foundType;
+
+            try {
+                foundType = findContentType(entry.weightCheck.bodyBuffer);
+            
+                if (!entry.contentType || entry.contentType === '') {
+                    if (foundType === null) {
+                        debug('ContentType is empty for file %s', entry.url);
+                    } else {
+                        debug('ContentType is empty for file %s. It should be %s.', entry.url, foundType.mimes[0]);
+                        entry.oldContentType = null;
+                        rewriteContentType(entry, foundType);
+                    }
+                } else {
+                    if (foundType !== null && foundType.mimes.indexOf(entry.contentType) === -1) {
+                        debug('ContentType %s is wrong for %s. It should be %s.', entry.contentType, entry.url, foundType.mimes[0]);
+                        entry.oldContentType = entry.contentType;
+                        rewriteContentType(entry, foundType);
+                    }
+                }
+
+            } catch(err) {
+                debug('Error while analyzing the contentType of %s', entry.url);
+                debug(err);
+            }
+        }
+
+        deferred.resolve(entry);
+
+        return deferred.promise;
+    }
+
+    function findContentType(bodyBuffer) {
+        var bodyStr = bodyBuffer.toString();
+
+        if (isJpg(bodyBuffer)) {
+            return contentTypes.jpeg;
+        }
+
+        if (isPng(bodyBuffer)) {
+            return contentTypes.png;
+        }
+
+        // https://github.com/sindresorhus/is-svg/issues/7
+        if (/<svg/.test(bodyStr) && isSvg(bodyStr)) {
+            return contentTypes.svg;
+        }
+
+        if (isGif(bodyBuffer)) {
+            return contentTypes.gif;
+        }
+
+        if (isWoff(bodyBuffer)) {
+            return contentTypes.woff;
+        }
+
+        if (isWoff2(bodyBuffer)) {
+            return contentTypes.woff2;
+        }
+
+        if (isOtf(bodyBuffer)) {
+            return contentTypes.otf;
+        }
+
+        if (isTtf(bodyBuffer)) {
+            return contentTypes.ttf;
+        }
+
+        if (isEot(bodyBuffer)) {
+            return contentTypes.eot;
+        }
+
+        return null;
+    }
+
+
+    function rewriteContentType(entry, contentTypeObj) {
+        delete(entry.isHTML);
+        delete(entry.isXML);
+        delete(entry.isCSS);
+        delete(entry.isJS);
+        delete(entry.isJSON);
+        delete(entry.isImage);
+        delete(entry.isSVG);
+        delete(entry.isVideo);
+        delete(entry.isWebFont);
+        delete(entry.isTTF);
+        delete(entry.isFavicon);
+
+        entry.contentType = contentTypeObj.mimes[0];
+        contentTypeObj.updateFn(entry);
+    }
+
+    var contentTypes = {
+        jpeg: {
+            mimes: ['image/jpeg'],
+            updateFn: function(entry) {
+                entry.type = 'image';
+                entry.isImage = true;
+            }
+        },
+        png: {
+            mimes: ['image/png'],
+            updateFn: function(entry) {
+                entry.type = 'image';
+                entry.isImage = true;
+            }
+        },
+        svg: {
+            mimes: ['image/svg+xml'],
+            updateFn: function(entry) {
+                entry.type = 'image';
+                entry.isImage = true;
+                entry.isSVG = true;
+            }
+        },
+        gif: {
+            mimes: ['image/gif'],
+            updateFn: function(entry) {
+                entry.type = 'image';
+                entry.isImage = true;
+            }
+        },
+        woff: {
+            mimes: ['application/x-font-woff', 'application/font-woff', 'font/woff'],
+            updateFn: function(entry) {
+                entry.type = 'webfont';
+                entry.isWebFont = true;
+            }
+        },
+        woff2: {
+            mimes: ['font/woff2', 'application/x-font-woff2', 'application/font-woff2'],
+            updateFn: function(entry) {
+                entry.type = 'webfont';
+                entry.isWebFont = true;
+            }
+        },
+        otf: {
+            mimes: ['application/x-font-otf', 'font/otf', 'font/opentype', 'application/x-font-opentype'],
+            updateFn: function(entry) {
+                entry.type = 'webfont';
+                entry.isWebFont = true;
+            }
+        },
+        ttf: {
+            mimes: ['application/x-font-ttf', 'font/ttf', 'application/x-font-truetype'],
+            updateFn: function(entry) {
+                entry.type = 'webfont';
+                entry.isWebFont = true;
+            }
+        },
+        eot: {
+            mimes: ['application/vnd.ms-fontobject', 'font/eot'],
+            updateFn: function(entry) {
+                entry.type = 'webfont';
+                entry.isWebFont = true;
+            }
+        }
+    };
+    
+    return {
+        checkContentType: checkContentType,
+        findContentType: findContentType
+    };
+};
+
+module.exports = new ContentTypeChecker();

lib/tools/redownload/fileMinifier.js

@@ -11,22 +11,24 @@ var FileMinifier = function() {
     function minifyFile(entry) {
         var deferred = Q.defer();
 
-        if (!entry.weightCheck || !entry.weightCheck.body) {
+        if (!entry.weightCheck || !entry.weightCheck.bodyBuffer) {
             // No valid file available
             deferred.resolve(entry);
             return deferred.promise;
         }
 
         var fileSize = entry.weightCheck.uncompressedSize;
+        var bodyString = entry.weightCheck.bodyBuffer.toString();
+
         debug('Let\'s try to optimize %s', entry.url);
         debug('Current file size is %d', fileSize);
         var startTime = Date.now();
 
-        if (entry.isJS && !isKnownAsMinified(entry.url) && !looksAlreadyMinified(entry.weightCheck.body)) {
+        if (entry.isJS && !isKnownAsMinified(entry.url) && !looksAlreadyMinified(bodyString)) {
 
             debug('File is a JS');
 
-            return minifyJs(entry.weightCheck.body)
+            return minifyJs(bodyString)
 
             .then(function(newFile) {
                 if (!newFile) {
@@ -58,7 +60,7 @@ var FileMinifier = function() {
 
             debug('File is a CSS');
 
-            return minifyCss(entry.weightCheck.body)
+            return minifyCss(entry.weightCheck.bodyBuffer.toString())
 
             .then(function(newFile) {
                 if (!newFile) {
@@ -91,7 +93,7 @@ var FileMinifier = function() {
 
             debug('File is an HTML');
 
-            return minifyHtml(entry.weightCheck.body)
+            return minifyHtml(entry.weightCheck.bodyBuffer.toString())
 
             .then(function(newFile) {
                 if (!newFile) {
@@ -121,7 +123,7 @@ var FileMinifier = function() {
             });
 
         } else {
-            debug('Not minfiable type or already minified');
+            debug('Not minifiable type or already minified');
             deferred.resolve(entry);
         }
 
@@ -146,7 +148,16 @@ var FileMinifier = function() {
         .delay(1)
         .then(splittedUglifyStep2)
         .delay(1)
-        .then(splittedUglifyStep3)
+        .then(function(ast) {
+            // Only do the compression step for smaller files
+            // otherwise it can take a very long time compared to the gain
+            if (body.length < 200*1024) {
+                return splittedUglifyStep3(ast);
+            } else {
+                debug('Skipping step 3 because the file is too big (%d bytes)!', body.length);
+                return ast;
+            }
+        })
         .delay(1)
         .then(splittedUglifyStep4)
         .delay(1)
@@ -285,7 +296,7 @@ var FileMinifier = function() {
         return deferred.promise;
     }
 
-    // Avoid loosing time trying to compress some JS libraries known as already compressed
+    // Avoid losing time trying to compress some JS libraries known as already compressed
     function isKnownAsMinified(url) {
         var result = false;
 
@@ -311,13 +322,13 @@ var FileMinifier = function() {
         return result;
     }
 
-    // Avoid loosing tome trying to compress JS files if they alreay look minified
+    // Avoid losing some trying to compress JS files if they already look minified
     // by counting the number of lines compared to the total size.
-    // Less than 1000kb per line is suspicious
+    // Less than 2KB per line is suspicious
     function looksAlreadyMinified(code) {
         var linesCount = code.split(/\r\n|\r|\n/).length;
         var linesRatio = code.length / linesCount;
-        var looksMinified = linesRatio > 1024;
+        var looksMinified = linesRatio > 2 * 1024;
         
         debug('Lines ratio is %d bytes per line', Math.round(linesRatio));
         debug(looksMinified ? 'It looks already minified' : 'It doesn\'t look minified');
@@ -340,4 +351,4 @@ var FileMinifier = function() {
     };
 };
 
-module.exports = new FileMinifier();
+module.exports = new FileMinifier();

lib/tools/redownload/fontAnalyzer.js

@@ -0,0 +1,388 @@
+var debug = require('debug')('ylt:fontAnalyzer');
+
+var Q           = require('q');
+var fontkit     = require('fontkit');
+
+var FontAnalyzer = function() {
+
+    function analyzeFont(entry, charsListOnPage) {
+        var deferred = Q.defer();
+        
+        if (!entry.weightCheck || !entry.weightCheck.bodyBuffer) {
+            // No valid file available
+            deferred.resolve(entry);
+            return deferred.promise;
+        }
+
+        var fileSize = entry.weightCheck.uncompressedSize;
+
+        if (entry.isWebFont) {
+            debug('File %s is a font. Let\'s have a look inside!', entry.url);
+            
+            getMetricsFromFont(entry, charsListOnPage)
+
+            .then(function(fontMetrics) {
+                entry.fontMetrics = fontMetrics;
+                deferred.resolve(entry);
+            })
+
+            .fail(function(error) {
+                debug('Could not open the font: %s', error);
+                deferred.resolve(entry);
+            });
+
+        } else {
+            deferred.resolve(entry);
+        }
+        
+        return deferred.promise;
+    }
+
+    function getMetricsFromFont(entry, charsListOnPage) {
+        var deferred = Q.defer();
+        
+        try {
+            var startTime = Date.now();
+            var font = fontkit.create(entry.weightCheck.bodyBuffer);
+
+            var result = {
+                name: font.fullName || font.postscriptName || font.familyName,
+                numGlyphs: font.numGlyphs,
+                averageGlyphComplexity: getAverageGlyphComplexity(font),
+                compressedWeight: entry.weightCheck.afterCompression || entry.weightCheck.bodySize,
+                unicodeRanges: readUnicodeRanges(font.characterSet, charsListOnPage),
+                numGlyphsInCommonWithPageContent: countPossiblyUsedGlyphs(getCharacterSetAsString(font.characterSet), charsListOnPage)
+            };
+
+            var endTime = Date.now();
+            debug('Font analysis took %dms', endTime - startTime);
+
+            // Mark fonts that are not used on the page (#224)
+            var fontIsUsed = false;
+            for (var range in result.unicodeRanges) {
+                if (result.unicodeRanges[range].numGlyphsInCommonWithPageContent > 0) {
+                    fontIsUsed = true;
+                    break;
+                }
+            }
+            result.isUsed = fontIsUsed;
+
+            deferred.resolve(result);
+        } catch(error) {
+            deferred.reject(error);
+        }
+
+        return deferred.promise;
+    }
+
+    // Reads the number of vector commands (complexity) needed to render glyphs and
+    // returns the average. Only first 100 glyphes are tested, otherwise it would take tool long;
+    function getAverageGlyphComplexity(font) {
+        var max = Math.min(font.numGlyphs, 100);
+
+        var totalPathsCommands = 0;
+        for (var i = 0; i < max; i++) {
+            totalPathsCommands += font.getGlyph(i).path.commands.length;
+        }
+        return Math.round(totalPathsCommands / max * 10) / 10;
+    }
+
+    function readUnicodeRanges(charsetInFont, charsListOnPage) {
+        var ranges = {};
+
+        // Assign a range to each char found in the font
+        charsetInFont.forEach(function(char) {
+            
+            var currentRange = getUnicodeRangeFromChar(char);
+            var currentRangeName = currentRange.name;
+
+            if (!ranges[currentRangeName]) {
+                // Cloning the object
+                ranges[currentRangeName] = Object.assign({}, currentRange);
+            }
+
+            if (!ranges[currentRangeName].charset) {
+                ranges[currentRangeName].charset = '';
+            }
+
+            ranges[currentRangeName].charset += String.fromCharCode(char);
+        });
+
+        var range;
+        var expectedLength;
+        var actualLength;
+
+        for (var rangeName in ranges) {
+            /*jshint loopfunc: true */
+
+            range = ranges[rangeName];
+
+            // Estimate if range is used, based on the characters found in the page
+            range.numGlyphsInCommonWithPageContent = countPossiblyUsedGlyphs(range.charset, charsListOnPage);
+
+            // Calculate coverage
+            if (rangeName !== 'Others') {
+                expectedLength = range.rangeEnd - range.rangeStart + 1;
+                actualLength = range.charset.length;
+                range.coverage = Math.min(actualLength / expectedLength, 1);
+            }
+        }
+
+        return ranges;
+    }
+
+    function countPossiblyUsedGlyphs(charsetInFont, charsListOnPage) {
+        var count = 0;
+        charsListOnPage.split('').forEach(function(char) {
+            if (charsetInFont.indexOf(char) >= 0) {
+                count ++;
+            }
+        });
+        return count;
+    }
+
+    function getCharacterSetAsString(characterSet) {
+        var str = '';
+        characterSet.forEach(function(charCode) {
+            str += String.fromCharCode(charCode);
+        });
+        return str;
+    }
+
+    function getUnicodeRangeFromChar(char) {
+        return UNICODE_RANGES.find(function(range) {
+            return (char >= range.rangeStart && char <= range.rangeEnd);
+        }) || {name: 'Others'};
+    }
+
+
+    var UNICODE_RANGES = [
+        {
+            name: 'Basic Latin',
+            rangeStart: 0x0020,
+            rangeEnd: 0x007F
+        },
+        {
+            name: 'Latin-1 Supplement',
+            rangeStart: 0x00A0,
+            rangeEnd: 0x00FF
+        },
+        {
+            name: 'Latin Extended',
+            rangeStart: 0x0100,
+            rangeEnd: 0x024F
+        },
+        {
+            name: 'IPA Extensions',
+            rangeStart: 0x0250,
+            rangeEnd: 0x02AF
+        },
+        {
+            name: 'Greek and Coptic',
+            rangeStart: 0x0370,
+            rangeEnd: 0x03FF
+        },
+        {
+            name: 'Cyrillic',
+            rangeStart: 0x0400,
+            rangeEnd: 0x052F
+        },
+        {
+            name: 'Armenian',
+            rangeStart: 0x0530,
+            rangeEnd: 0x058F
+        },
+        {
+            name: 'Hebrew',
+            rangeStart: 0x0590,
+            rangeEnd: 0x05FF
+        },
+        {
+            name: 'Arabic',
+            rangeStart: 0x0600,
+            rangeEnd: 0x06FF
+        },
+        {
+            name: 'Syriac',
+            rangeStart: 0x0700,
+            rangeEnd: 0x074F
+        },
+        {
+            name: 'Thaana',
+            rangeStart: 0x0780,
+            rangeEnd: 0x07BF
+        },
+        {
+            name: 'Devanagari',
+            rangeStart: 0x0900,
+            rangeEnd: 0x097F
+        },
+        {
+            name: 'Bengali',
+            rangeStart: 0x0980,
+            rangeEnd: 0x09FF
+        },
+        {
+            name: 'Gurmukhi',
+            rangeStart: 0x0A00,
+            rangeEnd: 0x0A7F
+        },
+        {
+            name: 'Gujarati',
+            rangeStart: 0x0A80,
+            rangeEnd: 0x0AFF
+        },
+        {
+            name: 'Oriya',
+            rangeStart: 0x0B00,
+            rangeEnd: 0x0B7F
+        },
+        {
+            name: 'Tamil',
+            rangeStart: 0x0B80,
+            rangeEnd: 0x0BFF
+        },
+        {
+            name: 'Telugu',
+            rangeStart: 0x0C00,
+            rangeEnd: 0x0C7F
+        },
+        {
+            name: 'Kannada',
+            rangeStart: 0x0C80,
+            rangeEnd: 0x0CFF
+        },
+        {
+            name: 'Malayalam',
+            rangeStart: 0x0D00,
+            rangeEnd: 0x0D7F
+        },
+        {
+            name: 'Sinhala',
+            rangeStart: 0x0D80,
+            rangeEnd: 0x0DFF
+        },
+        {
+            name: 'Thai',
+            rangeStart: 0x0E00,
+            rangeEnd: 0x0E7F
+        },
+        {
+            name: 'Lao',
+            rangeStart: 0x0E80,
+            rangeEnd: 0x0EFF
+        },
+        {
+            name: 'Tibetan',
+            rangeStart: 0x0F00,
+            rangeEnd: 0x0FFF
+        },
+        {
+            name: 'Myanmar',
+            rangeStart: 0x1000,
+            rangeEnd: 0x109F
+        },
+        {
+            name: 'Georgian',
+            rangeStart: 0x10A0,
+            rangeEnd: 0x10FF
+        },
+        {
+            name: 'Hangul Jamo',
+            rangeStart: 0x1100,
+            rangeEnd: 0x11FF
+        },
+        {
+            name: 'Ethiopic',
+            rangeStart: 0x1200,
+            rangeEnd: 0x137F
+        },
+        {
+            name: 'Cherokee',
+            rangeStart: 0x13A0,
+            rangeEnd: 0x13FF
+        },
+        {
+            name: 'Unified Canadian Aboriginal Syllabics',
+            rangeStart: 0x1400,
+            rangeEnd: 0x167F
+        },
+        {
+            name: 'Ogham',
+            rangeStart: 0x1680,
+            rangeEnd: 0x169F
+        },
+        {
+            name: 'Runic',
+            rangeStart: 0x16A0,
+            rangeEnd: 0x16FF
+        },
+        {
+            name: 'Tagalog',
+            rangeStart: 0x1700,
+            rangeEnd: 0x171F
+        },
+        {
+            name: 'Hanunoo',
+            rangeStart: 0x1720,
+            rangeEnd: 0x173F
+        },
+        {
+            name: 'Buhid',
+            rangeStart: 0x1740,
+            rangeEnd: 0x175F
+        },
+        {
+            name: 'Tagbanwa',
+            rangeStart: 0x1760,
+            rangeEnd: 0x177F
+        },
+        {
+            name: 'Khmer',
+            rangeStart: 0x1780,
+            rangeEnd: 0x17FF
+        },
+        {
+            name: 'Mongolian',
+            rangeStart: 0x1800,
+            rangeEnd: 0x18AF
+        },
+        {
+            name: 'Limbu',
+            rangeStart: 0x1900,
+            rangeEnd: 0x194F
+        },
+        {
+            name: 'Tai Le',
+            rangeStart: 0x1950,
+            rangeEnd: 0x197F
+        },
+        {
+            name: 'Hiragana',
+            rangeStart: 0x3040,
+            rangeEnd: 0x309F
+        },
+        {
+            name: 'Katakana',
+            rangeStart: 0x30A0,
+            rangeEnd: 0x30FF
+        },
+        {
+            name: 'Bopomofo',
+            rangeStart: 0x3100,
+            rangeEnd: 0x312F
+        }
+    ];
+
+    return {
+        analyzeFont: analyzeFont,
+        getMetricsFromFont: getMetricsFromFont,
+        readUnicodeRanges: readUnicodeRanges,
+        getAverageGlyphComplexity: getAverageGlyphComplexity,
+        countPossiblyUsedGlyphs: countPossiblyUsedGlyphs,
+        getCharacterSetAsString: getCharacterSetAsString,
+        getUnicodeRangeFromChar: getUnicodeRangeFromChar
+    };
+};
+
+module.exports = new FontAnalyzer();

lib/tools/redownload/gzipCompressor.js

@@ -6,6 +6,7 @@ var zlib    = require('zlib');
 var GzipCompressor = function() {
 
     function compressFile(entry) {
+        debug('Entering compressFile');
         return gzipUncompressedFile(entry)
 
         .then(gzipOptimizedFile);
@@ -15,12 +16,12 @@ var GzipCompressor = function() {
     function gzipUncompressedFile(entry) {
         var deferred = Q.defer();
 
-        if (entryTypeCanBeGzipped(entry) && entry.weightCheck && !entry.weightCheck.isCompressed && entry.weightCheck.body) {
+        if (entryTypeCanBeGzipped(entry) && entry.weightCheck && !entry.weightCheck.isCompressed && entry.weightCheck.bodyBuffer) {
             debug('Compression missing, trying to gzip file %s', entry.url);
 
             var uncompressedSize = entry.weightCheck.uncompressedSize;
 
-            zlib.gzip(new Buffer(entry.weightCheck.body, 'utf8'), function(err, buffer) {
+            zlib.gzip(entry.weightCheck.bodyBuffer, function(err, buffer) {
                 if (err) {
                     debug('Could not compress uncompressed file with gzip');
                     debug(err);
@@ -41,7 +42,7 @@ var GzipCompressor = function() {
                 }
             });
         } else {
-            
+            debug('Compression not needed');
             deferred.resolve(entry);
         }
 
@@ -73,6 +74,7 @@ var GzipCompressor = function() {
                 }
             });
         } else {
+            debug('Compressing optimized file not needed');
             deferred.resolve(entry);
         }
 

lib/tools/redownload/imageOptimizer.js

@@ -1,8 +1,10 @@
-var debug = require('debug')('ylt:imageOptimizer');
-
-var Q           = require('q');
-var Imagemin    = require('imagemin');
-var jpegoptim   = require('imagemin-jpegoptim');
+var debug               = require('debug')('ylt:imageOptimizer');
+var Q                   = require('q');
+var imagemin            = require('imagemin');
+var imageminJpegtran    = require('imagemin-jpegtran');
+var imageminJpegoptim   = require('imagemin-jpegoptim');
+var imageminOptipng     = require('imagemin-optipng');
+var imageminSvgo        = require('imagemin-svgo');
 
 var ImageOptimizer = function() {
 
@@ -12,7 +14,7 @@ var ImageOptimizer = function() {
     function optimizeImage(entry) {
         var deferred = Q.defer();
 
-        if (!entry.weightCheck || !entry.weightCheck.body) {
+        if (!entry.weightCheck || !entry.weightCheck.bodyBuffer) {
             // No valid file available
             deferred.resolve(entry);
             return deferred.promise;
@@ -26,7 +28,7 @@ var ImageOptimizer = function() {
             debug('File is a JPEG');
 
             // Starting softly with a lossless compression
-            return compressJpegLosslessly(new Buffer(entry.weightCheck.body, 'binary'))
+            return compressJpegLosslessly(entry.weightCheck.bodyBuffer)
 
             .then(function(newFile) {
                 if (!newFile) {
@@ -34,7 +36,7 @@ var ImageOptimizer = function() {
                     return entry;
                 }
 
-                var newFileSize = newFile.contents.length;
+                var newFileSize = newFile.length;
 
                 debug('JPEG lossless compression complete for %s', entry.url);
                 
@@ -46,7 +48,7 @@ var ImageOptimizer = function() {
 
 
                 // Now let's compress lossy to MAX_JPEG_QUALITY
-                return compressJpegLossly(new Buffer(entry.weightCheck.body, 'binary'));
+                return compressJpegLossly(entry.weightCheck.bodyBuffer);
             })
             
             .then(function(newFile) {
@@ -55,11 +57,13 @@ var ImageOptimizer = function() {
                     return entry;
                 }
 
-                var newFileSize = newFile.contents.length;
+                var newFileSize = newFile.length;
 
                 debug('JPEG lossy compression complete for %s', entry.url);
 
-                if (gainIsEnough(fileSize, newFileSize)) {
+                if (entry.weightCheck.lossless && entry.weightCheck.lossless < newFileSize) {
+                    debug('Lossy compression is not as good as lossless compression. Skipping the lossy.');
+                } else if (gainIsEnough(fileSize, newFileSize)) {
                     
                     if (entry.weightCheck.isOptimized !== false || newFileSize < entry.weightCheck.lossless) {
                         entry.weightCheck.optimized = newFileSize;
@@ -82,7 +86,7 @@ var ImageOptimizer = function() {
             debug('File is a PNG');
 
             // Starting softly with a lossless compression
-            return compressPngLosslessly(new Buffer(entry.weightCheck.body, 'binary'))
+            return compressPngLosslessly(entry.weightCheck.bodyBuffer)
 
             .then(function(newFile) {
                 if (!newFile) {
@@ -90,7 +94,7 @@ var ImageOptimizer = function() {
                     return entry;
                 }
                 
-                var newFileSize = newFile.contents.length;
+                var newFileSize = newFile.length;
 
                 debug('PNG lossless compression complete for %s', entry.url);
                 
@@ -116,7 +120,7 @@ var ImageOptimizer = function() {
             debug('File is an SVG');
 
             // Starting softly with a lossless compression
-            return compressSvgLosslessly(new Buffer(entry.weightCheck.body, 'utf8'))
+            return compressSvgLosslessly(entry.weightCheck.bodyBuffer)
 
             .then(function(newFile) {
                 if (!newFile) {
@@ -124,12 +128,12 @@ var ImageOptimizer = function() {
                     return entry;
                 }
 
-                var newFileSize = newFile.contents.length;
+                var newFileSize = newFile.length;
 
                 debug('SVG lossless compression complete for %s', entry.url);
                 
                 if (gainIsEnough(fileSize, newFileSize)) {
-                    entry.weightCheck.bodyAfterOptimization = newFile.contents.toString();
+                    entry.weightCheck.bodyAfterOptimization = newFile.toString();
                     entry.weightCheck.lossless = entry.weightCheck.optimized = newFileSize;
                     entry.weightCheck.isOptimized = false;
                     debug('Filesize is %d bytes smaller (-%d%)', fileSize - newFileSize, Math.round((fileSize - newFileSize) * 100 / fileSize));
@@ -194,35 +198,31 @@ var ImageOptimizer = function() {
 
         var engine;
         if (type === 'jpeg' && !lossy) {
-            engine = Imagemin.jpegtran();
+            engine = imageminJpegtran({progressive: true});
         } else if (type === 'jpeg' && lossy) {
-            engine = jpegoptim({max: MAX_JPEG_QUALITY});
+            engine = imageminJpegoptim({progressive: true, max: MAX_JPEG_QUALITY});
         } else if (type === 'png' && !lossy) {
-            engine = Imagemin.optipng({optimizationLevel: OPTIPNG_COMPRESSION_LEVEL});
+            engine = imageminOptipng({optimizationLevel: OPTIPNG_COMPRESSION_LEVEL});
         } else if (type === 'svg' && !lossy) {
-            engine = Imagemin.svgo();
+            engine = imageminSvgo({ plugins: [ { removeUselessDefs: false } ] });
         } else {
             deferred.reject('No optimization engine found for imagemin');
         }
 
-        try {
 
-            new Imagemin()
-                .src(imageBody)
-                .use(engine)
-                .run(function (err, files) {
-                    if (err) {
-                        deferred.reject(err);
-                    } else {
-                        deferred.resolve(files[0]);
-                        var endTime = Date.now();
-                        debug('Optimization for %s took %d ms', type, endTime - startTime);
-                    }
-                });
+        imagemin.buffer(imageBody, {use: engine})
 
-            } catch(err) {
-                deferred.reject(err);
-            }
+        .then(function(file) {
+            var endTime = Date.now();
+            debug('Optimization for %s took %d ms', type, endTime - startTime);
+            deferred.resolve(file);
+        })
+
+        .catch(function(err) {
+            debug('Optimisation failed:');
+            debug(err);
+            deferred.resolve();
+        });
 
         return deferred.promise;
     }

lib/tools/redownload/redownload.js

@@ -0,0 +1,802 @@
+/*
+ * Redownloading every files after Phantomas has finished
+ * Checks weight and every kind of compression
+ *
+ */
+
+/*jshint -W069 */
+
+var debug               = require('debug')('ylt:redownload');
+var Q                   = require('q');
+var http                = require('http');
+var zlib                = require('zlib');
+var async               = require('async');
+var request             = require('request');
+var md5                 = require('md5');
+
+var imageOptimizer      = require('./imageOptimizer');
+var fileMinifier        = require('./fileMinifier');
+var gzipCompressor      = require('./gzipCompressor');
+var contentTypeChecker  = require('./contentTypeChecker');
+var fontAnalyzer        = require('./fontAnalyzer');
+
+
+var Redownload = function() {
+
+    var MAX_PARALLEL_DOWNLOADS = 10;
+    var REQUEST_TIMEOUT = 15000; // 15 seconds
+
+
+    // This function will re-download every asset and check if it could be optimized
+    function recheckAllFiles(data) {
+        var startTime = Date.now();
+        debug('Redownload started');
+        var deferred = Q.defer();
+
+        var requestsList = JSON.parse(data.toolsResults.phantomas.offenders.requestsList);
+        delete data.toolsResults.phantomas.metrics.requestsList;
+        delete data.toolsResults.phantomas.offenders.requestsList;
+
+        var httpAuth = null;
+        if (data.params && data.params.options && data.params.options.authUser && data.params.options.authPass) {
+            httpAuth = {
+                username: data.params.options.authUser,
+                password: data.params.options.authPass
+            };
+        }
+
+        var proxy = null;
+        if (data.params && data.params.options && data.params.options.proxy) {
+            proxy = data.params.options.proxy;
+            if (proxy.indexOf('http:') === -1) {
+                proxy = 'http://' + proxy;
+            }
+        }
+
+        // Prevent a bug with the font analyzer on empty pages
+        var differentCharacters = '';
+        if (data.toolsResults.phantomas.offenders.differentCharacters && data.toolsResults.phantomas.offenders.differentCharacters.length > 0) {
+            differentCharacters = data.toolsResults.phantomas.offenders.differentCharacters[0];
+        }
+
+        // Transform every request into a download function with a callback when done
+        var redownloadList = requestsList.map(function(entry) {
+            return function(callback) {
+                
+                redownloadEntry(entry, httpAuth, proxy)
+
+                .then(contentTypeChecker.checkContentType)
+
+                .then(imageOptimizer.optimizeImage)
+
+                .then(fileMinifier.minifyFile)
+
+                .then(gzipCompressor.compressFile)
+
+                .then(function(entry) {
+                    return fontAnalyzer.analyzeFont(entry, differentCharacters);
+                })
+
+                .then(function(newEntry) {
+                    debug('File %s - Redownloaded, optimized, minified, compressed, analyzed: done', entry.url);
+                    callback(null, newEntry);
+                })
+
+                .fail(function(err) {
+                    callback(err);
+                });
+            };
+        });
+
+        // Lanch all redownload functions and wait for completion
+        async.parallelLimit(redownloadList, MAX_PARALLEL_DOWNLOADS, function(err, results) {
+            
+            if (err) {
+                debug(err);
+                deferred.reject(err);
+            } else {
+
+                debug('All files checked');
+                endTime = Date.now();
+                debug('Redownload took %d ms', endTime - startTime);
+                
+                var metrics = {};
+                var offenders = {};
+
+                // Remove unused fonts that a normal browser would not download (fix #224)
+                results = results.filter(function(result) {
+                    if (result && result.fontMetrics) {
+                        return result.fontMetrics.isUsed !== false;
+                    }
+                    return true;
+                });
+
+                // Count requests
+                offenders.totalRequests = listRequestsByType(results);
+                metrics.totalRequests = offenders.totalRequests.total;
+
+
+                // Remove unwanted responses (redownload failed, about:blank)
+                results = results.filter(function(result) {
+                    return (result !== null && result.weightCheck && result.url.indexOf('about:blank') !== 0);
+                });
+
+
+                // Wrong contentType
+                offenders.incorrectContentTypes = listIncorrectContentTypes(results);
+                metrics.incorrectContentTypes = offenders.incorrectContentTypes.length;
+
+                // Total weight
+                offenders.totalWeight = listRequestWeight(results);
+                metrics.totalWeight = offenders.totalWeight.totalWeight;
+
+                // Empty files
+                offenders.emptyRequests = listEmptyRequests(results);
+                metrics.emptyRequests = offenders.emptyRequests.length;
+
+
+                // Now remove unwanted responses (redirections and empty files)
+                results = results.filter(function(result) {
+                    return ((result.status < 300 || result.status >= 400) && result.weightCheck.bodySize > 0);
+                });
+
+
+                // Image compression
+                offenders.imageOptimization = listImageNotOptimized(results);
+                metrics.imageOptimization = offenders.imageOptimization.totalGain;
+
+                // File minification
+                offenders.fileMinification = listFilesNotMinified(results);
+                metrics.fileMinification = offenders.fileMinification.totalGain;
+
+                // Gzip compression
+                offenders.gzipCompression = listFilesNotGzipped(results);
+                metrics.gzipCompression = offenders.gzipCompression.totalGain;
+
+                // Small requests
+                offenders.smallRequests = listSmallRequests(results);
+                metrics.smallRequests = offenders.smallRequests.total;
+
+                // Detect identical files
+                offenders.identicalFiles = listIdenticalFiles(results);
+                metrics.identicalFiles = offenders.identicalFiles.avoidableRequests;
+
+                // Fonts count
+                offenders.fontsCount = listFonts(results);
+                metrics.fontsCount = offenders.fontsCount.count;
+
+                // Heavy fonts
+                offenders.heavyFonts = listHeavyFonts(results);
+                metrics.heavyFonts = offenders.heavyFonts.totalGain;
+
+                // Unused Unicode ranges
+                offenders.unusedUnicodeRanges = listUnusedUnicodeRanges(results);
+                metrics.unusedUnicodeRanges = offenders.unusedUnicodeRanges.count;
+
+
+                data.toolsResults.redownload = {
+                    metrics: metrics,
+                    offenders: offenders
+                };
+
+                deferred.resolve(data);
+            }
+        });
+
+        return deferred.promise;
+    }
+
+    function listIncorrectContentTypes(requests) {
+        var results = [];
+        
+        requests.forEach(function(req) {
+            if (req.oldContentType || req.oldContentType === null) {
+                results.push({
+                    url: req.url,
+                    current: req.oldContentType,
+                    correct: req.contentType
+                });
+            }
+        });
+
+        return results;
+    }
+
+    function listEmptyRequests(requests) {
+        var results = [];
+        
+        requests.forEach(function(req) {
+            var weight = req.weightCheck.bodySize || req.contentLength;
+
+            if (weight === 0 && req.method === 'GET' && req.status === 200) {
+                results.push(req.url);
+            }
+        });
+
+        return results;
+    }
+
+    function listRequestWeight(requests) {
+        var results = {
+            totalWeight: 0,
+            byType: {
+                html: {
+                    totalWeight: 0,
+                    requests: []
+                },
+                css: {
+                    totalWeight: 0,
+                    requests: []
+                },
+                js: {
+                    totalWeight: 0,
+                    requests: []
+                },
+                json: {
+                    totalWeight: 0,
+                    requests: []
+                },
+                image: {
+                    totalWeight: 0,
+                    requests: []
+                },
+                video: {
+                    totalWeight: 0,
+                    requests: []
+                },
+                webfont: {
+                    totalWeight: 0,
+                    requests: []
+                },
+                other: {
+                    totalWeight: 0,
+                    requests: []
+                }
+            }
+        };
+
+        requests.forEach(function(req) {
+            var weight = ((typeof req.weightCheck.bodySize === 'number') ? req.weightCheck.bodySize + req.weightCheck.headersSize : req.contentLength) || 0;
+            var type = req.type || 'other';
+            type = (results.byType[type]) ? type : 'other';
+
+            results.totalWeight += weight;
+            results.byType[type].totalWeight += weight;
+
+            results.byType[type].requests.push({
+                url: req.url,
+                weight: weight
+            });
+        });
+
+        return results;
+    }
+
+
+    function listImageNotOptimized(requests) {
+        var results = {
+            totalGain: 0,
+            images: []
+        };
+
+        requests.forEach(function(req) {
+            if (req.weightCheck.bodySize > 0 && imageOptimizer.entryTypeCanBeOptimized(req) && req.weightCheck.isOptimized === false) {
+                var before = req.weightCheck.afterCompression || req.weightCheck.bodySize;
+                var after = req.weightCheck.afterOptimizationAndCompression || req.weightCheck.optimized;
+                var gain = before - after;
+
+                if (gain > 200) {
+                    results.totalGain += gain;
+
+                    results.images.push({
+                        url: req.url,
+                        original: req.weightCheck.bodySize,
+                        isCompressed: req.weightCheck.isCompressed,
+                        afterCompression: req.weightCheck.afterCompression,
+                        afterOptimizationAndCompression: req.weightCheck.afterOptimizationAndCompression,
+                        lossless: req.weightCheck.lossless,
+                        lossy: req.weightCheck.lossy,
+                        gain: gain
+                    });
+                }
+            }
+        });
+
+        return results;
+    }
+
+
+    function listFilesNotMinified(requests) {
+        var results = {
+            totalGain: 0,
+            files: []
+        };
+
+        requests.forEach(function(req) {
+            if (req.weightCheck.bodySize > 0 && fileMinifier.entryTypeCanBeMinified(req) && req.weightCheck.isOptimized === false) {
+                var before = req.weightCheck.afterCompression || req.weightCheck.bodySize;
+                var after = req.weightCheck.afterOptimizationAndCompression || req.weightCheck.optimized;
+                var gain = before - after;
+
+                if (gain > 200) {
+                    results.totalGain += gain;
+
+                    results.files.push({
+                        url: req.url,
+                        original: req.weightCheck.bodySize,
+                        isCompressed: req.weightCheck.isCompressed,
+                        afterCompression: req.weightCheck.afterCompression,
+                        afterOptimizationAndCompression: req.weightCheck.afterOptimizationAndCompression,
+                        optimized: req.weightCheck.optimized,
+                        gain: gain
+                    });
+                }
+            }
+        });
+
+        return results;
+    }
+
+    function listFilesNotGzipped(requests) {
+        var results = {
+            totalGain: 0,
+            files: []
+        };
+
+        requests.forEach(function(req) {
+            if (req.weightCheck.uncompressedSize && req.weightCheck.isCompressed === false && req.weightCheck.afterCompression) {
+                var gain = req.weightCheck.uncompressedSize - req.weightCheck.afterCompression;
+
+                results.totalGain += gain;
+
+                results.files.push({
+                    url: req.url,
+                    original: req.weightCheck.uncompressedSize,
+                    gzipped: req.weightCheck.afterCompression,
+                    gain: gain
+                });
+            }
+        });
+
+        return results;
+    }
+
+    function listRequestsByType(requests) {
+        var results = {
+            total: 0,
+            byType: {
+                html: [],
+                css: [],
+                js: [],
+                json: [],
+                image: [],
+                video: [],
+                webfont: [],
+                other: []
+            }
+        };
+
+        requests.forEach(function(req) {
+            if (req.url !== 'about:blank') {
+                var type = req.type || 'other';
+                type = (results.byType[type]) ? type : 'other';
+                
+                results.byType[type].push(req.url);
+                results.total ++;
+            }
+        });
+
+        return results;
+    }
+
+    function listSmallRequests(requests) {
+        var results = {
+            total: 0,
+            byType: {
+                css: [],
+                js: [],
+                image: []
+            }
+        };
+
+        requests.forEach(function(req) {
+            if (req.weightCheck.bodySize > 0 && req.weightCheck.bodySize < 2048) {
+                if (req.isCSS || req.isJS || req.isImage) {
+                    results.byType[req.type].push({
+                        url: req.url,
+                        size: req.weightCheck.bodySize
+                    });
+                    results.total ++;
+                }
+            }
+        });
+
+        return results;
+    }
+
+    function listIdenticalFiles(requests) {
+        var hashes = {};
+        var list = [];
+        var avoidableRequestsCount = 0;
+
+        requests.forEach(function(req) {
+            var requestHash = md5(req.weightCheck.bodyBuffer);
+            
+            // Try to exclude tracking pixels
+            if (req.weightCheck.bodySize < 80 && req.type === 'image') {
+                return;
+            }
+
+            if (!hashes[requestHash]) {
+                hashes[requestHash] = {
+                    weight: req.weightCheck.bodySize,
+                    urls: []
+                };
+            }
+            
+            if (hashes[requestHash].urls.indexOf(req.url) === -1) {
+                hashes[requestHash].urls.push(req.url);
+            }
+        });
+
+        for (var hash in hashes) {
+            if (hashes[hash].urls.length > 1) {
+                list.push(hashes[hash]);
+                avoidableRequestsCount += hashes[hash].urls.length - 1;
+            }
+        }
+
+        return {
+            avoidableRequests: avoidableRequestsCount,
+            count: list.length,
+            list: list
+        };
+    }
+
+    function listFonts(requests) {
+        var list = [];
+
+        requests.forEach(function(req) {
+            if (req.isWebFont) {
+                list.push({
+                    url: req.url,
+                    size: req.weightCheck.bodySize
+                });
+            }
+        });
+
+        return {
+            count: list.length,
+            list: list
+        };
+    }
+
+    function listHeavyFonts(requests) {
+        var list = [];
+        var totalGain = 0;
+        var heavyFontsCount = 0;
+        var MAX_FONT_WEIGHT = 40 * 1024;
+
+        requests.forEach(function(req) {
+            if (req.isWebFont && req.fontMetrics) {
+                list.push({
+                    url: req.url,
+                    weight: req.weightCheck.bodySize,
+                    numGlyphs: req.fontMetrics.numGlyphs,
+                    averageGlyphComplexity: req.fontMetrics.averageGlyphComplexity
+                });
+                if (req.weightCheck.bodySize > MAX_FONT_WEIGHT) {
+                    totalGain += req.weightCheck.bodySize - MAX_FONT_WEIGHT;
+                    heavyFontsCount ++;
+                }
+            }
+        });
+
+        return {
+            count: heavyFontsCount,
+            fonts: list,
+            totalGain: totalGain
+        };
+    }
+
+    function listUnusedUnicodeRanges(requests) {
+        var list = [];
+        var unusedUnicodeRanges = 0;
+
+        requests.forEach(function(req) {
+            if (req.isWebFont && req.fontMetrics && req.fontMetrics.unicodeRanges) {
+                var ranges = [];
+                var others = null;
+                var rangeNames = Object.keys(req.fontMetrics.unicodeRanges);
+                var unicodePointsCount = 0;
+                var unusedRangesInFont = 0;
+                
+                rangeNames.forEach(function(rangeName) {
+                    var range = req.fontMetrics.unicodeRanges[rangeName];
+
+                    // Exclude "Others"
+                    if (rangeName === 'Others') {
+                        if (range.numGlyphsInCommonWithPageContent === 0 && range.charset.length > 50) {
+                            range.underused = true;
+                            unusedRangesInFont ++;
+                        }
+
+                        unicodePointsCount += range.charset.length;
+                        others = range;
+                    } else if (range.charset.length > 0) {
+                        // Now lets detect if the current Unicode range is unused.
+                        // Reminder: range.coverage = glyphs declared in this range, divided by the range size
+                        if (range.coverage > 0.25 && range.numGlyphsInCommonWithPageContent === 0) {
+                            range.underused = true;
+                            unusedRangesInFont ++;
+                        }
+
+                        unicodePointsCount += range.charset.length;
+                        ranges.push(range);
+                    }
+                });
+
+
+                // Detect if it's a icons font : if more than 90% of the icons are
+                // in the "Others", it looks like one.
+                if (others && others.charset.length / req.fontMetrics.numGlyphs > 0.9) {
+                    
+                    list.push({
+                        url: req.url,
+                        weight: req.weightCheck.bodySize,
+                        isIconFont: true,
+                        glyphs: req.fontMetrics.numGlyphs,
+                        numGlyphsInCommonWithPageContent: req.fontMetrics.numGlyphsInCommonWithPageContent
+                    });
+
+                    // And if less than 5% of the icons are used, let's report it as underused
+                    if (others && others.numGlyphsInCommonWithPageContent / others.charset.length <= 0.05) {
+                        unusedRangesInFont = 1;
+                    }
+
+                // Not an icons font
+                } else {
+                    if (others) {
+                        // Insert back "Others" at the end of the list
+                        ranges.push(others);
+                    }
+
+                    var ligaturesOrHiddenChars = req.fontMetrics.numGlyphs - unicodePointsCount;
+                    if (ligaturesOrHiddenChars > 25) {
+                        unusedUnicodeRanges ++;
+                    }
+
+                    list.push({
+                        url: req.url,
+                        weight: req.weightCheck.bodySize,
+                        isIconFont: false,
+                        unicodeRanges: ranges,
+                        ligaturesOrHiddenChars: ligaturesOrHiddenChars 
+                    });
+                }
+
+                unusedUnicodeRanges += unusedRangesInFont;
+            }
+        });
+
+        return {
+            count: unusedUnicodeRanges,
+            fonts: list
+        };
+    }
+
+
+    function redownloadEntry(entry, httpAuth, proxy) {
+        var deferred = Q.defer();
+        
+        function downloadError(message) {
+            debug('Could not download %s Error: %s', entry.url, message);
+            entry.weightCheck = {
+                message: message
+            };
+            deferred.resolve(entry);
+        }
+
+        // Not downloaded again but will be counted in totalWeight
+        function notDownloadableFile(message) {
+            entry.weightCheck = {
+                message: message
+            };
+            deferred.resolve(entry);
+        }
+
+        // Not counted in totalWeight
+        function unwantedFile(message) {
+            debug(message);
+            deferred.resolve(entry);
+        }
+
+        if (entry.method !== 'GET') {
+            notDownloadableFile('only downloading GET');
+            return deferred.promise;
+        }
+
+        if (entry.status !== 200) {
+            unwantedFile('only downloading requests with status code 200');
+            return deferred.promise;
+        }
+
+        if (entry.url === 'about:blank') {
+            unwantedFile('not downloading about:blank');
+            return deferred.promise;
+        }
+
+
+        debug('Downloading %s', entry.url);
+
+        // Always add a gzip header before sending, in case the server listens to it
+        var reqHeaders = entry.requestHeaders;
+        reqHeaders['Accept-Encoding'] = 'gzip, deflate';
+        reqHeaders['Connection'] = 'keep-alive';
+
+        var requestOptions = {
+            method: entry.method,
+            url: entry.url,
+            headers: reqHeaders,
+            timeout: REQUEST_TIMEOUT,
+            proxy: proxy
+        };
+
+        // Basic auth
+        if (httpAuth) {
+            requestOptions.auth = {
+                user: httpAuth.username,
+                pass: httpAuth.password,
+                sendImmediately: false // Tries a first time without auth, wait for a 401 error before resending
+            };
+        }
+
+        download(requestOptions, entry.contentType, function(error, result) {
+            if (error) {
+                if (error.code === 'ETIMEDOUT') {
+                    downloadError('timeout after ' + REQUEST_TIMEOUT + 'ms');
+                } else {
+                    downloadError('error while downloading: ' + error.code);
+                }
+                return;
+            }
+                
+            debug('%s downloaded correctly', entry.url);
+
+            entry.weightCheck = result;
+            deferred.resolve(entry);
+        });
+
+        return deferred.promise;
+    }
+
+    // Inspired by https://github.com/cvan/fastHAR-api/blob/10cec585/app.js
+    function download(requestOptions, contentType, callback) {
+
+        var statusCode;
+
+        try {
+
+            request(requestOptions)
+
+            .on('response', function(res) {
+                
+                // Raw headers were added in NodeJS v0.12
+                // (https://github.com/joyent/node/issues/4844), but let's
+                // reconstruct them for backwards compatibility.
+                var rawHeaders = ('HTTP/' + res.httpVersion + ' ' + res.statusCode +
+                                  ' ' + http.STATUS_CODES[res.statusCode] + '\r\n');
+                Object.keys(res.headers).forEach(function(headerKey) {
+                    rawHeaders += headerKey + ': ' + res.headers[headerKey] + '\r\n';
+                });
+                rawHeaders += '\r\n';
+
+                var uncompressedSize = 0;  // size after uncompression
+                var bodySize = 0;  // bytes size over the wire
+                var bodyChunks = [];  // an array of buffers
+                var isCompressed = false;
+
+                function tally() {
+
+                    if (statusCode !== 200) {
+                        callback({code: statusCode});
+                        return;
+                    }
+
+                    var body = Buffer.concat(bodyChunks);
+
+                    var result = {
+                        bodyBuffer: body,
+                        headersSize: Buffer.byteLength(rawHeaders, 'utf8'),
+                        bodySize: bodySize,
+                        isCompressed: isCompressed,
+                        uncompressedSize: uncompressedSize
+                    };
+
+                    callback(null, result);
+                }
+
+                switch (res.headers['content-encoding']) {
+                    case 'gzip':
+
+                        var gzip = zlib.createGunzip();
+
+                        gzip.on('data', function (data) {
+
+                            bodyChunks.push(data);
+                            uncompressedSize += data.length;
+                        }).on('end', function () {
+                            isCompressed = true;
+                            tally();
+                        }).on('error', function(err) {
+                            debug('Error while decoding %s', requestOptions.url);
+                            debug(err);
+                            callback(err);
+                        });
+
+                        res.on('data', function (data) {
+                            bodySize += data.length;
+                        }).pipe(gzip);
+
+                        break;
+                    case 'deflate':
+                        res.setEncoding('utf8');
+
+                        var deflate = zlib.createInflate();
+
+                        deflate.on('data', function (data) {
+                            bodyChunks.push(data);
+                            uncompressedSize += data.length;
+                        }).on('end', function () {
+                            isCompressed = true;
+                            tally();
+                        }).on('error', function(err) {
+                            debug('Error while decoding %s', requestOptions.url);
+                            debug(err);
+                            callback(err);
+                        });
+
+                        res.on('data', function (data) {
+                            bodySize += data.length;
+                        }).pipe(deflate);
+
+                        break;
+                    default:
+                        res.on('data', function (data) {
+                            bodyChunks.push(data);
+                            uncompressedSize += data.length;
+                            bodySize += data.length;
+                        }).on('end', function () {
+                            tally();
+                        });
+
+                        break;
+                }
+            })
+
+            .on('response', function(response) {
+                statusCode = response.statusCode;
+            })
+
+            .on('error', function(err) {
+                debug('Error while downloading %s', requestOptions.url);
+                debug(err);
+                callback(err);
+            });
+
+        } catch(err) {
+            debug('Error while downloading %s', requestOptions.url);
+            debug(err);
+            callback(err);
+        }
+    }
+
+    return {
+        recheckAllFiles: recheckAllFiles,
+        listRequestWeight: listRequestWeight,
+        redownloadEntry: redownloadEntry,
+        download: download
+    };
+};
+
+module.exports = new Redownload();

lib/tools/weightChecker/weightChecker.js

@@ -1,501 +0,0 @@
-/*
- * Redownloading every files after Phantomas has finished
- * Checks weight and every kind of compression
- *
- */
-
-
-var debug           = require('debug')('ylt:weightChecker');
-var Q               = require('q');
-var http            = require('http');
-var zlib            = require('zlib');
-var async           = require('async');
-var request         = require('request');
-
-var imageOptimizer  = require('./imageOptimizer');
-var fileMinifier    = require('./fileMinifier');
-var gzipCompressor  = require('./gzipCompressor');
-
-
-var WeightChecker = function() {
-
-    var MAX_PARALLEL_DOWNLOADS = 10;
-    var REQUEST_TIMEOUT = 15000; // 15 seconds
-
-
-    // This function will re-download every asset and check if it could be optimized
-    function recheckAllFiles(data) {
-        var startTime = Date.now();
-        debug('Redownload started');
-        var deferred = Q.defer();
-
-        var requestsList = JSON.parse(data.toolsResults.phantomas.offenders.requestsList);
-        delete data.toolsResults.phantomas.metrics.requestsList;
-        delete data.toolsResults.phantomas.offenders.requestsList;
-
-        // Transform every request into a download function with a callback when done
-        var redownloadList = requestsList.map(function(entry) {
-            return function(callback) {
-                
-                redownloadEntry(entry)
-
-                .then(imageOptimizer.optimizeImage)
-
-                .then(fileMinifier.minifyFile)
-
-                .then(gzipCompressor.compressFile)
-
-                .then(function(newEntry) {
-                    callback(null, newEntry);
-                })
-
-                .fail(function(err) {
-                    callback(err);
-                });
-            };
-        });
-
-        // Lanch all redownload functions and wait for completion
-        async.parallelLimit(redownloadList, MAX_PARALLEL_DOWNLOADS, function(err, results) {
-            
-            if (err) {
-                debug(err);
-                deferred.reject(err);
-            } else {
-
-                debug('All files checked');
-                endTime = Date.now();
-                debug('Redownload took %d ms', endTime - startTime);
-                
-                var metrics = {};
-                var offenders = {};
-
-                // Count requests
-                offenders.totalRequests = listRequestsByType(results);
-                metrics.totalRequests = offenders.totalRequests.total;
-
-                // Remove unwanted requests (redirections, about:blank)
-                results = results.filter(function(result) {
-                    return (result !== null && result.weightCheck && result.weightCheck.bodySize > 0);
-                });
-
-
-                // Total weight
-                offenders.totalWeight = listRequestWeight(results);
-                metrics.totalWeight = offenders.totalWeight.totalWeight;
-
-                // Image compression
-                offenders.imageOptimization = listImageNotOptimized(results);
-                metrics.imageOptimization = offenders.imageOptimization.totalGain;
-
-                // File minification
-                offenders.fileMinification = listFilesNotMinified(results);
-                metrics.fileMinification = offenders.fileMinification.totalGain;
-
-                // Gzip compression
-                offenders.gzipCompression = listFilesNotGzipped(results);
-                metrics.gzipCompression = offenders.gzipCompression.totalGain;
-
-                // Small requests
-                offenders.smallRequests = listSmallRequests(results);
-                metrics.smallRequests = offenders.smallRequests.total;
-
-                data.toolsResults.weightChecker = {
-                    metrics: metrics,
-                    offenders: offenders
-                };
-
-                deferred.resolve(data);
-            }
-        });
-
-        return deferred.promise;
-    }
-
-
-    function listRequestWeight(requests) {
-        var results = {
-            totalWeight: 0,
-            byType: {
-                html: {
-                    totalWeight: 0,
-                    requests: []
-                },
-                css: {
-                    totalWeight: 0,
-                    requests: []
-                },
-                js: {
-                    totalWeight: 0,
-                    requests: []
-                },
-                json: {
-                    totalWeight: 0,
-                    requests: []
-                },
-                image: {
-                    totalWeight: 0,
-                    requests: []
-                },
-                video: {
-                    totalWeight: 0,
-                    requests: []
-                },
-                webfont: {
-                    totalWeight: 0,
-                    requests: []
-                },
-                other: {
-                    totalWeight: 0,
-                    requests: []
-                }
-            }
-        };
-
-        requests.forEach(function(req) {
-            var weight = ((typeof req.weightCheck.bodySize === 'number') ? req.weightCheck.bodySize + req.weightCheck.headersSize : req.contentLength) || 0;
-            var type = req.type || 'other';
-
-            results.totalWeight += weight;
-            results.byType[type].totalWeight += weight;
-
-            results.byType[type].requests.push({
-                url: req.url,
-                weight: weight
-            });
-        });
-
-        return results;
-    }
-
-
-    function listImageNotOptimized(requests) {
-        var results = {
-            totalGain: 0,
-            images: []
-        };
-
-        requests.forEach(function(req) {
-            if (req.weightCheck.bodySize > 0 && imageOptimizer.entryTypeCanBeOptimized(req) && req.weightCheck.isOptimized === false) {
-                var before = req.weightCheck.afterCompression || req.weightCheck.bodySize;
-                var after = req.weightCheck.afterOptimizationAndCompression || req.weightCheck.optimized;
-                var gain = before - after;
-
-                if (gain > 200) {
-                    results.totalGain += gain;
-
-                    results.images.push({
-                        url: req.url,
-                        original: req.weightCheck.bodySize,
-                        isCompressed: req.weightCheck.isCompressed,
-                        afterCompression: req.weightCheck.afterCompression,
-                        afterOptimizationAndCompression: req.weightCheck.afterOptimizationAndCompression,
-                        lossless: req.weightCheck.lossless,
-                        lossy: req.weightCheck.lossy,
-                        gain: gain
-                    });
-                }
-            }
-        });
-
-        return results;
-    }
-
-
-    function listFilesNotMinified(requests) {
-        var results = {
-            totalGain: 0,
-            files: []
-        };
-
-        requests.forEach(function(req) {
-            if (req.weightCheck.bodySize > 0 && fileMinifier.entryTypeCanBeMinified(req) && req.weightCheck.isOptimized === false) {
-                var before = req.weightCheck.afterCompression || req.weightCheck.bodySize;
-                var after = req.weightCheck.afterOptimizationAndCompression || req.weightCheck.optimized;
-                var gain = before - after;
-
-                if (gain > 200) {
-                    results.totalGain += gain;
-
-                    results.files.push({
-                        url: req.url,
-                        original: req.weightCheck.bodySize,
-                        isCompressed: req.weightCheck.isCompressed,
-                        afterCompression: req.weightCheck.afterCompression,
-                        afterOptimizationAndCompression: req.weightCheck.afterOptimizationAndCompression,
-                        optimized: req.weightCheck.optimized,
-                        gain: gain
-                    });
-                }
-            }
-        });
-
-        return results;
-    }
-
-    function listFilesNotGzipped(requests) {
-        var results = {
-            totalGain: 0,
-            files: []
-        };
-
-        requests.forEach(function(req) {
-            if (req.weightCheck.uncompressedSize && req.weightCheck.isCompressed === false && req.weightCheck.afterCompression) {
-                var gain = req.weightCheck.uncompressedSize - req.weightCheck.afterCompression;
-
-                results.totalGain += gain;
-
-                results.files.push({
-                    url: req.url,
-                    original: req.weightCheck.uncompressedSize,
-                    gzipped: req.weightCheck.afterCompression,
-                    gain: gain
-                });
-            }
-        });
-
-        return results;
-    }
-
-    function listRequestsByType(requests) {
-        var results = {
-            total: 0,
-            byType: {
-                html: [],
-                css: [],
-                js: [],
-                json: [],
-                image: [],
-                video: [],
-                webfont: [],
-                other: []
-            }
-        };
-
-        requests.forEach(function(req) {
-            if (req.url !== 'about:blank') {
-                var type = req.type || 'other';
-                results.byType[type].push(req.url);
-                results.total ++;
-            }
-        });
-
-        return results;
-    }
-
-    function listSmallRequests(requests) {
-        var results = {
-            total: 0,
-            byType: {
-                css: [],
-                js: [],
-                image: []
-            }
-        };
-
-        requests.forEach(function(req) {
-            if (req.weightCheck.bodySize > 0 && req.weightCheck.bodySize < 2048) {
-                if (req.isCSS || req.isJS || req.isImage) {
-                    results.byType[req.type].push({
-                        url: req.url,
-                        size: req.weightCheck.bodySize
-                    });
-                    results.total ++;
-                }
-            }
-        });
-
-        return results;
-    }
-
-
-    function redownloadEntry(entry) {
-        var deferred = Q.defer();
-        
-        function downloadError(message) {
-            debug('Could not download %s Error: %s', entry.url, message);
-            entry.weightCheck = {
-                message: message
-            };
-            deferred.resolve(entry);
-        }
-
-        // Not downloaded again but will be counted in totalWeight
-        function notDownloadableFile(message) {
-            entry.weightCheck = {
-                message: message
-            };
-            deferred.resolve(entry);
-        }
-
-        // Not counted in totalWeight
-        function unwantedFile(message) {
-            debug(message);
-            deferred.resolve(entry);
-        }
-
-        if (entry.method !== 'GET') {
-            notDownloadableFile('only downloading GET');
-            return deferred.promise;
-        }
-
-        if (entry.status !== 200) {
-            unwantedFile('only downloading requests with status code 200');
-            return deferred.promise;
-        }
-
-        if (entry.url === 'about:blank') {
-            unwantedFile('not downloading about:blank');
-            return deferred.promise;
-        }
-
-
-        debug('Downloading %s', entry.url);
-
-        // Always add a gzip header before sending, in case the server listens to it
-        var reqHeaders = entry.requestHeaders;
-        reqHeaders['Accept-Encoding'] = 'gzip, deflate';
-
-        var requestOptions = {
-            method: entry.method,
-            url: entry.url,
-            headers: reqHeaders,
-            timeout: REQUEST_TIMEOUT
-        };
-
-        download(requestOptions, entry.contentType, function(error, result) {
-            if (error) {
-                if (error.code === 'ETIMEDOUT') {
-                    downloadError('timeout after ' + REQUEST_TIMEOUT + 'ms');
-                } else {
-                    downloadError('error while downloading: ' + error.code);
-                }
-                return;
-            }
-                
-            debug('%s downloaded correctly', entry.url);
-
-            entry.weightCheck = result;
-            deferred.resolve(entry);
-        });
-
-        return deferred.promise;
-    }
-
-    // Inspired by https://github.com/cvan/fastHAR-api/blob/10cec585/app.js
-    function download(requestOptions, contentType, callback) {
-
-        var statusCode;
-
-        request(requestOptions)
-
-        .on('response', function(res) {
-            
-            // Raw headers were added in NodeJS v0.12
-            // (https://github.com/joyent/node/issues/4844), but let's
-            // reconstruct them for backwards compatibility.
-            var rawHeaders = ('HTTP/' + res.httpVersion + ' ' + res.statusCode +
-                              ' ' + http.STATUS_CODES[res.statusCode] + '\r\n');
-            Object.keys(res.headers).forEach(function(headerKey) {
-                rawHeaders += headerKey + ': ' + res.headers[headerKey] + '\r\n';
-            });
-            rawHeaders += '\r\n';
-
-            var uncompressedSize = 0;  // size after uncompression
-            var bodySize = 0;  // bytes size over the wire
-            var body = '';  // plain text body (after uncompressing gzip/deflate)
-            var isCompressed = false;
-
-            function tally() {
-
-                if (statusCode !== 200) {
-                    callback({code: statusCode});
-                    return;
-                }
-
-                var result = {
-                    body: body,
-                    headersSize: Buffer.byteLength(rawHeaders, 'utf8'),
-                    bodySize: bodySize,
-                    isCompressed: isCompressed,
-                    uncompressedSize: uncompressedSize
-                };
-
-                callback(null, result);
-            }
-
-            switch (res.headers['content-encoding']) {
-                case 'gzip':
-
-                    var gzip = zlib.createGunzip();
-
-                    gzip.on('data', function (data) {
-                        body += data;
-                        uncompressedSize += data.length;
-                    }).on('end', function () {
-                        isCompressed = true;
-                        tally();
-                    }).on('error', function(err) {
-                        debug(err);
-                    });
-
-                    res.on('data', function (data) {
-                        bodySize += data.length;
-                    }).pipe(gzip);
-
-                    break;
-                case 'deflate':
-                    res.setEncoding('utf8');
-
-                    var deflate = zlib.createInflate();
-
-                    deflate.on('data', function (data) {
-                        body += data;
-                        uncompressedSize += data.length;
-                    }).on('end', function () {
-                        isCompressed = true;
-                        tally();
-                    }).on('error', function(err) {
-                        debug(err);
-                    });
-
-                    res.on('data', function (data) {
-                        bodySize += data.length;
-                    }).pipe(deflate);
-
-                    break;
-                default:
-                    if (contentType === 'image/jpeg' || contentType === 'image/png') {
-                        res.setEncoding('binary');
-                    }
-
-                    res.on('data', function (data) {
-                        body += data;
-                        uncompressedSize += data.length;
-                        bodySize += data.length;
-                    }).on('end', function () {
-                        tally();
-                    });
-
-                    break;
-            }
-        })
-
-        .on('response', function(response) {
-            statusCode = response.statusCode;
-        })
-
-        .on('error', function(err) {
-            callback(err);
-        });
-    }
-
-    return {
-        recheckAllFiles: recheckAllFiles,
-        listRequestWeight: listRequestWeight,
-        redownloadEntry: redownloadEntry,
-        download: download
-    };
-};
-
-module.exports = new WeightChecker();

package.json

@@ -1,7 +1,13 @@
 {
   "name": "yellowlabtools",
-  "version": "1.7.0",
+  "version": "1.13.0",
   "description": "Online tool to audit a webpage for performance and front-end quality issues",
+  "license": "GPL-2.0",
+  "author": {
+    "name": "Gaël Métais",
+    "email": "gael@gaelmetais.com",
+    "url": "http://www.gaelmetais.com"
+  },
   "repository": {
     "type": "git",
     "url": "git://github.com/gmetais/YellowLabTools.git"
@@ -9,55 +15,95 @@
   "bin": {
     "yellowlabtools": "./bin/cli.js"
   },
+  "engines": {
+    "node": ">= 4.0"
+  },
   "main": "./lib/index.js",
   "dependencies": {
-    "async": "1.2.1",
-    "body-parser": "1.13.1",
-    "clean-css": "3.3.0",
-    "compression": "1.5.0",
-    "cors": "2.7.1",
-    "debug": "2.2.0",
-    "express": "4.12.4",
-    "imagemin": "3.2.0",
-    "imagemin-jpegoptim": "4.0.0",
-    "lwip": "0.0.7",
-    "meow": "3.3.0",
-    "minimize": "1.4.1",
-    "phantomas": "1.11.0",
-    "ps-node": "0.0.4",
-    "q": "1.4.1",
-    "rimraf": "2.4.0",
+    "angular": "1.6.8",
+    "angular-animate": "1.6.8",
+    "angular-chart.js": "1.1.1",
+    "angular-local-storage": "0.7.1",
+    "angular-resource": "1.6.8",
+    "angular-route": "1.6.8",
+    "angular-sanitize": "1.6.8",
+    "async": "2.6.0",
+    "body-parser": "1.18.2",
+    "chart.js": "2.7.1",
+    "clean-css": "4.1.9",
+    "color-diff": "1.1.0",
+    "compression": "1.7.1",
+    "cors": "2.8.4",
+    "css-mq-parser": "0.0.3",
+    "debug": "3.1.0",
+    "easyxml": "2.0.1",
+    "ejs": "2.5.7",
+    "express": "4.16.2",
+    "fontkit": "1.7.7",
+    "imagemin": "5.3.1",
+    "imagemin-jpegoptim": "5.2.0",
+    "imagemin-jpegtran": "5.0.2",
+    "imagemin-optipng": "5.2.1",
+    "imagemin-svgo": "6.0.0",
+    "is-eot": "1.0.0",
+    "is-gif": "1.0.0",
+    "is-http2": "1.2.0",
+    "is-jpg": "1.0.0",
+    "is-otf": "0.1.2",
+    "is-png": "1.1.0",
+    "is-svg": "2.1.0",
+    "is-ttf": "0.2.2",
+    "is-woff": "1.0.3",
+    "is-woff2": "1.0.0",
+    "jimp": "0.2.28",
+    "md5": "2.2.1",
+    "meow": "3.7.0",
+    "minimize": "2.1.0",
+    "parse-color": "1.0.0",
+    "phantomas": "1.19.0",
+    "ps-node": "0.1.6",
+    "q": "1.5.1",
+    "request": "2.83.0",
+    "rimraf": "2.6.2",
     "temporary": "0.0.8",
-    "uglify-js": "2.4.23"
+    "try-thread-sleep": "1.0.2",
+    "uglify-js": "2.8.29"
   },
   "devDependencies": {
-    "chai": "^3.0.0",
-    "grunt": "^0.4.5",
-    "grunt-blanket": "^0.0.8",
-    "grunt-contrib-clean": "^0.6.0",
-    "grunt-contrib-concat": "^0.5.1",
-    "grunt-contrib-copy": "^0.8.0",
-    "grunt-contrib-cssmin": "^0.12.3",
-    "grunt-contrib-htmlmin": "^0.4.0",
-    "grunt-contrib-jshint": "^0.11.2",
-    "grunt-contrib-less": "^1.0.1",
-    "grunt-contrib-uglify": "^0.9.1",
-    "grunt-env": "^0.4.4",
-    "grunt-express": "^1.4.1",
-    "grunt-filerev": "^2.3.1",
-    "grunt-inline-angular-templates": "^0.1.5",
-    "grunt-line-remover": "^0.0.2",
-    "grunt-mocha-test": "^0.12.7",
-    "grunt-replace": "^0.9.2",
-    "grunt-usemin": "^3.0.0",
-    "grunt-webfont": "^0.5.3",
-    "matchdep": "^0.3.0",
-    "mocha": "^2.2.5",
-    "request": "^2.58.0",
-    "sinon": "^1.15.3",
-    "sinon-chai": "^2.8.0"
+    "chai": "~4.1.2",
+    "grunt": "~1.0.1",
+    "grunt-contrib-clean": "~1.1.0",
+    "grunt-contrib-concat": "~1.0.1",
+    "grunt-contrib-copy": "~1.0.0",
+    "grunt-contrib-cssmin": "~2.2.1",
+    "grunt-contrib-htmlmin": "~2.4.0",
+    "grunt-contrib-jshint": "~1.1.0",
+    "grunt-contrib-less": "~1.4.1",
+    "grunt-contrib-uglify": "~3.3.0",
+    "grunt-contrib-watch": "~1.0.0",
+    "grunt-env": "~0.4.4",
+    "grunt-express": "~1.4.1",
+    "grunt-filerev": "~2.3.1",
+    "grunt-inline-angular-templates": "~0.1.5",
+    "grunt-mocha-test": "~0.13.3",
+    "grunt-parallel": "~0.5.1",
+    "grunt-replace": "~1.0.1",
+    "grunt-usemin": "~3.1.1",
+    "grunt-webfont": "~1.6.0",
+    "matchdep": "~2.0.0",
+    "mocha": "~4.1.0",
+    "sinon": "~4.1.6",
+    "sinon-chai": "~2.14.0"
   },
   "scripts": {
     "test": "grunt test"
-  }
+  },
+  "keywords": [
+    "performance",
+    "metrics",
+    "webperf",
+    "pagespeed",
+    "budget",
+    "phantomas"
+  ]
 }

server_config/maintenance.js

@@ -0,0 +1,13 @@
+var express                 = require('express');
+var app                     = express();
+var server                  = require('http').createServer(app);
+
+var settings                = require('./settings.json');
+
+app.all('*', function(req, res) {
+    res.status(500).send('YellowLabTools is in maintenance. It should come back soon with a new version!');
+});
+
+server.listen(settings.serverPort, function() {
+    console.log('Maintenance mode started on port %d', server.address().port);
+});

server_config/server_install.sh

@@ -2,27 +2,22 @@
 
 # APT-GET
 sudo apt-get update
-sudo apt-get install lsb-release libfontconfig1 libfreetype6 -y --force-yes
-sudo apt-get install curl git python-software-properties build-essential make g++ -y --force-yes
+sudo apt-get install lsb-release libfontconfig1 libfreetype6 libjpeg-dev -y --force-yes > /dev/null 2>&1
+sudo apt-get install curl git software-properties-common build-essential make g++ -y --force-yes > /dev/null 2>&1
 
 # Installation of NodeJS
-sudo add-apt-repository -y ppa:chris-lea/node.js
-sudo apt-get update
-sudo apt-get install -y nodejs
+curl -sL https://deb.nodesource.com/setup_4.x | sudo -E bash -
+sudo apt-get install -y nodejs > /dev/null 2>&1
 source ~/.profile
 
 # Installation of some packages globally
-npm install bower forever grunt-cli -g
+npm install forever grunt-cli -g
 source ~/.profile
 
 # Installation of YellowLabTools
-sudo mkdir /space
-sudo chown $USER /space
-cd /space
-git clone https://github.com/gmetais/YellowLabTools.git --branch master
-cd YellowLabTools
-npm install
-bower install --config.interactive=false --allow-root
+sudo chown -R $USER /space
+cd /space/YellowLabTools
+npm install || exit 1
 
 # Front-end compilation
 grunt build

server_config/server_update.sh

@@ -2,23 +2,23 @@
 
 cd /space/YellowLabTools
 
-# Stop the server
+# Stop the server and start the maintenance mode
 forever stopall
+forever start server_config/maintenance.js
 
 # Keep the settings.json file
 git stash
 git pull
 git stash pop
 
-# In case something was added in package.json or bower.json
+# In case something was added in package.json
 rm -rf node_modules
-npm install
-rm -rf bower_components
-bower install --config.interactive=false --allow-root
+npm install || exit 1
 
 # Front-end compilation
 rm -rf front/build
 grunt build
 
-# Restart the server
+# Stop the maintenance mode and restart the server
+forever stopall
 NODE_ENV=production forever start -c "node --stack-size=262000" bin/server.js

server_config/settings-prod.json

@@ -1,10 +1,13 @@
 {
     "serverPort": 80,
+    "phantomasEngine": "webkit",
     "googleAnalyticsId": "",
-
+    "screenshotWidth": 400,
+    "baseUrl": "/",
     "authorizedKeys": {
         
     },
     "maxAnonymousRunsPerDay": 99999999,
-    "maxAnonymousCallsPerDay": 99999999
+    "maxAnonymousCallsPerDay": 99999999,
+    "blockedUrls": []
 }

server_config/settings.json

@@ -1,10 +1,13 @@
 {
     "serverPort": 8383,
+    "phantomasEngine": "webkit",
     "googleAnalyticsId": "",
-
+    "screenshotWidth": 400,
+    "baseUrl": "/",
     "authorizedKeys": {
         
     },
     "maxAnonymousRunsPerDay": 99999999,
-    "maxAnonymousCallsPerDay": 99999999
+    "maxAnonymousCallsPerDay": 99999999,
+    "blockedUrls": []
 }

test/api/apiTest.js

@@ -31,6 +31,7 @@ describe('api', function() {
             },
             json: true,
             headers: {
+                'Content-Type': 'application/json',
                 'X-Api-Key': 'invalid'
             }
         }, function(error, response, body) {
@@ -49,10 +50,12 @@ describe('api', function() {
             method: 'POST',
             url: serverUrl + '/api/runs',
             body: {
-                url: ''
+                url: '',
+                waitForResponse: true
             },
             json: true,
             headers: {
+                'Content-Type': 'application/json',
                 'X-Api-Key': Object.keys(config.authorizedKeys)[0]
             }
         }, function(error, response, body) {
@@ -76,6 +79,7 @@ describe('api', function() {
             },
             json: true,
             headers: {
+                'Content-Type': 'application/json',
                 'X-Api-Key': Object.keys(config.authorizedKeys)[0]
             }
         }, function(error, response, body) {
@@ -99,12 +103,13 @@ describe('api', function() {
                 screenshot: true,
                 device: 'tablet',
                 //waitForSelector: '*',
-                cookie: 'foo=bar',
+                cookie: 'foo=bar;domain=google.com',
                 authUser: 'joe',
                 authPass: 'secret'
             },
             json: true,
             headers: {
+                'Content-Type': 'application/json',
                 'X-Api-Key': Object.keys(config.authorizedKeys)[0]
             }
         }, function(error, response, body) {
@@ -133,6 +138,7 @@ describe('api', function() {
             },
             json: true,
             headers: {
+                'Content-Type': 'application/json',
                 'X-Api-Key': Object.keys(config.authorizedKeys)[0]
             }
         }, function(error, response, body) {
@@ -165,14 +171,13 @@ describe('api', function() {
                 body.should.have.a.property('rules').that.is.an('object');
                 body.should.have.a.property('toolsResults').that.is.an('object');
 
-                // javascriptExecutionTree should only be filled if option jsTimeline is true
                 body.should.have.a.property('javascriptExecutionTree').that.is.an('object');
-                body.javascriptExecutionTree.should.deep.equal({});
+                body.javascriptExecutionTree.should.not.deep.equal({});
 
                 // Check if settings are correctly sent and retrieved
                 body.params.options.should.have.a.property('device').that.equals('tablet');
                 //body.params.options.should.have.a.property('waitForSelector').that.equals('*');
-                body.params.options.should.have.a.property('cookie').that.equals('foo=bar');
+                body.params.options.should.have.a.property('cookie').that.equals('foo=bar;domain=google.com');
                 body.params.options.should.have.a.property('authUser').that.equals('joe');
                 body.params.options.should.have.a.property('authPass').that.equals('secret');
 
@@ -182,7 +187,7 @@ describe('api', function() {
                 body.should.not.have.a.property('screenshotBuffer');
                 // Check if the screenshot url is here
                 body.should.have.a.property('screenshotUrl');
-                body.screenshotUrl.should.equal('/api/results/' + body.runId + '/screenshot.jpg');
+                body.screenshotUrl.should.equal('api/results/' + body.runId + '/screenshot.jpg');
 
                 screenshotUrl = body.screenshotUrl;
 
@@ -208,6 +213,7 @@ describe('api', function() {
             },
             json: true,
             headers: {
+                'Content-Type': 'application/json',
                 'X-Api-Key': Object.keys(config.authorizedKeys)[0]
             }
         }, function(error, response, body) {
@@ -232,6 +238,7 @@ describe('api', function() {
             url: serverUrl + '/api/runs/' + asyncRunId,
             json: true,
             headers: {
+                'Content-Type': 'application/json',
                 'X-Api-Key': Object.keys(config.authorizedKeys)[0]
             }
         }, function(error, response, body) {
@@ -653,7 +660,7 @@ describe('api', function() {
 
         request({
             method: 'GET',
-            url: serverUrl + screenshotUrl
+            url: serverUrl + '/' + screenshotUrl
         }, function(error, response, body) {
             if (!error && response.statusCode === 200) {
                 response.headers['content-type'].should.equal('image/jpeg');
@@ -680,4 +687,28 @@ describe('api', function() {
         });
     });
 
+    it('should refuse a query on a blocked Url', function(done) {
+        this.timeout(5000);
+
+        request({
+            method: 'POST',
+            url: serverUrl + '/api/runs',
+            body: {
+                url: 'http://www.test.com/something.html',
+                waitForResponse: false
+            },
+            json: true,
+            headers: {
+                'Content-Type': 'application/json',
+                'X-Api-Key': Object.keys(config.authorizedKeys)[0]
+            }
+        }, function(error, response, body) {
+            if (!error && response.statusCode === 403) {
+                done();
+            } else {
+                done(error || response.statusCode);
+            }
+        });
+    });
+
 });

test/api/screenshotHandlerTest.js

@@ -7,17 +7,17 @@ var path = require('path');
 describe('screenshotHandler', function() {
 
     var imagePath = path.join(__dirname, '../fixtures/logo-large.png');
-    var screenshot, lwipImage;
+    var screenshot, jimpImage;
 
-    
-    it('should open an image and return an lwip object', function(done) {
+
+    it('should open an image and return an jimp object', function(done) {
         ScreenshotHandler.openImage(imagePath)
             .then(function(image) {
-                lwipImage = image;
+                jimpImage = image;
 
-                lwipImage.should.be.an('object');
-                lwipImage.width().should.equal(620);
-                lwipImage.height().should.equal(104);
+                jimpImage.should.be.an('object');
+                jimpImage.bitmap.width.should.equal(620);
+                jimpImage.bitmap.height.should.equal(104);
 
                 done();
             })
@@ -26,14 +26,14 @@ describe('screenshotHandler', function() {
             });
     });
 
-    
-    it('should resize an lwip image', function(done) {
-        ScreenshotHandler.resizeImage(lwipImage, 310)
+
+    it('should resize an jimp image', function(done) {
+        ScreenshotHandler.resizeImage(jimpImage, 310)
             .then(function(image) {
-                lwipImage = image;
+                jimpImage = image;
 
-                lwipImage.width().should.equal(310);
-                lwipImage.height().should.equal(52);
+                jimpImage.bitmap.width.should.equal(310);
+                jimpImage.bitmap.height.should.equal(52);
 
                 done();
             })
@@ -43,8 +43,8 @@ describe('screenshotHandler', function() {
     });
 
 
-    it('should transform a lwip image into a buffer', function(done) {
-        ScreenshotHandler.toBuffer(lwipImage)
+    it('should transform a jimp image into a buffer', function(done) {
+        ScreenshotHandler.toBuffer(jimpImage)
             .then(function(buffer) {
                 buffer.should.be.an.instanceof(Buffer);
                 done();
@@ -125,4 +125,4 @@ describe('screenshotHandler', function() {
                 done(err);
             });
     });
-});
+});

test/blanket.js

@@ -1,7 +0,0 @@
-var path = require('path');
-var srcDir = path.join(__dirname, '..', 'server');
-
-require('blanket')({
-  // Only files that match the pattern will be instrumented
-  pattern: srcDir
-});

test/core/colorDiffTest.js

@@ -0,0 +1,92 @@
+var should = require('chai').should();
+var colorDiff = require('../../lib/tools/colorDiff');
+
+describe('colorDiff', function() {
+    
+    it('should parse offenders correctly', function() {
+        colorDiff.parseOffender('#000 (1 times)').should.equal('#000');
+        colorDiff.parseOffender('#5bc0de (2 times)').should.equal('#5bc0de');
+        colorDiff.parseOffender('rgba(0,0,0,0.075) (100 times)').should.equal('rgba(0,0,0,0.075)');
+        colorDiff.parseOffender('rgb(91,192,222) (1000 times)').should.equal('rgb(91,192,222)');
+    });
+
+    it('should parse colors correctly', function() {
+        colorDiff.parseColor('#000').should.deep.equal({R: 0, G: 0, B: 0, A: 1, Lab: {L: 0, a: 0, b: 0}, original: '#000'});
+        colorDiff.parseColor('rgba(255,255,255,0.5)').should.deep.equal({R: 255, G: 255, B: 255, A: 0.5, Lab: {L: 100, a: 0.00526049995830391, b: -0.010408184525267927}, original: 'rgba(255,255,255,0.5)'});
+    });
+
+    it('should not compare colors with different alpha', function() {
+        var color1 = colorDiff.parseColor("rgba(0, 0, 0, 1)");
+        var color2 = colorDiff.parseColor("rgba(0, 0, 0, 0.5)");
+
+        colorDiff.compareTwoColors(color1, color2).should.equal(false);
+    });
+
+    it('should find that two colors are similar', function() {
+        var color1 = colorDiff.parseColor("rgba(0, 0, 0, 1)");
+        var color2 = colorDiff.parseColor("rgba(0, 0, 1, 1)");
+
+        colorDiff.compareTwoColors(color1, color2).should.equal(true);
+    });
+
+    it('should find that two colors are similar even with alpha', function() {
+        var color1 = colorDiff.parseColor("rgba(0, 0, 0, 0.3)");
+        var color2 = colorDiff.parseColor("rgba(0, 0, 2, 0.3)");
+
+        colorDiff.compareTwoColors(color1, color2).should.equal(true);
+    });
+
+    it('should find that two colors are different', function() {
+        var color1 = colorDiff.parseColor("rgba(0, 0, 0, 1)");
+        var color2 = colorDiff.parseColor("rgba(99, 99, 99, 1)");
+
+        colorDiff.compareTwoColors(color1, color2).should.equal(false);
+    });
+
+    it('should compare all colors to each other', function() {
+        var colors = [
+            '#000 (1 times)',
+            '#5bc0de (2 times)',
+            'rgba(0,0,0,0.075) (100 times)',
+            'rgb(91,192,222) (1000 times)',
+            'rgba(0,0,2,1) (1 times)',
+            'rgba(99,99,99,1) (1 times)',
+            'rgba(100,100,100,1) (1 times)'
+        ];
+
+        var data = {
+            toolsResults: {
+                phantomas: {
+                    offenders: {
+                        cssColors: colors
+                    }
+                }
+            }
+        };
+
+        newData = colorDiff.compareAllColors(data);
+
+        newData.toolsResults.should.have.a.property('colorDiff');
+        newData.toolsResults.colorDiff.should.have.a.property('metrics');
+        newData.toolsResults.colorDiff.metrics.should.have.a.property('similarColors').that.equals(3);
+        newData.toolsResults.colorDiff.should.have.a.property('offenders');
+        newData.toolsResults.colorDiff.offenders.should.have.a.property('similarColors').that.deep.equals([
+            {
+                color1: '#000',
+                color2: 'rgba(0,0,2,1)',
+                isDark: true
+            },
+            {
+                color1: '#5bc0de',
+                color2: 'rgb(91,192,222)',
+                isDark: false
+            },
+            {
+                color1: 'rgba(99,99,99,1)',
+                color2: 'rgba(100,100,100,1)',
+                isDark: true
+            }
+        ]);
+    });
+
+});

test/core/contentTypeCheckerTest.js

@@ -0,0 +1,20 @@
+var should = require('chai').should();
+var contentTypeChecker = require('../../lib/tools/redownload/contentTypeChecker');
+var fs = require('fs');
+var path = require('path');
+
+describe('contentTypeChecker', function() {
+
+    var jpgImageContent = fs.readFileSync(path.resolve(__dirname, '../www/jpeg-image.jpg'));
+    var pngImageContent = fs.readFileSync(path.resolve(__dirname, '../www/png-image.png'));
+    var svgImageContent = fs.readFileSync(path.resolve(__dirname, '../www/svg-image.svg'));
+    var cssFileContent = fs.readFileSync(path.resolve(__dirname, '../www/unminified-stylesheet.css'));
+    
+    it('detect the right content type', function() {
+        contentTypeChecker.findContentType(jpgImageContent).mimes.should.deep.equal(['image/jpeg']);
+        contentTypeChecker.findContentType(pngImageContent).mimes.should.deep.equal(['image/png']);
+        contentTypeChecker.findContentType(svgImageContent).mimes.should.deep.equal(['image/svg+xml']);
+        should.equal(contentTypeChecker.findContentType(cssFileContent), null);
+    });
+
+});

test/core/customPoliciesTest.js

@@ -182,19 +182,19 @@ describe('customPolicies', function() {
     it('should grade correctly jQuery versions', function() {
 
         var versions = {
-            '1.2.9': 0,
-            '1.3.9': 0,
-            '1.4.4': 10,
-            '1.5.0': 20,
-            '1.6.3': 30,
-            '1.7.0': 40,
-            '1.8.3a': 50,
-            '1.9.2': 70,
-            '1.10.1': 90,
-            '2.0.0-rc1': 90,
-            '1.11.1': 100,
-            '2.1.1-beta1': 100,
-            '3.0.0': 100
+            '1.7.0': 0,
+            '1.10.1': 20,
+            '1.10.3a': 20,
+            '2.0.0-rc1': 20,
+            '1.11.1': 30,
+            '2.1.1-beta1': 30,
+            '1.12.1': 40,
+            '2.2.1': 40,
+            '3.0.1': 50,
+            '3.1.0': 70,
+            '3.2.1': 90,
+            '3.3.1': 100,
+            '3.5.0': 100
         };
 
         for (var version in versions) {

test/core/fileMinifierTest.js

@@ -1,5 +1,5 @@
 var should = require('chai').should();
-var fileMinifier = require('../../lib/tools/weightChecker/fileMinifier');
+var fileMinifier = require('../../lib/tools/redownload/fileMinifier');
 var fs = require('fs');
 var path = require('path');
 
@@ -37,7 +37,7 @@ describe('fileMinifier', function() {
             type: 'js',
             contentLength: 999,
             weightCheck: {
-                body: fileContent.toString('utf8'),
+                bodyBuffer: fileContent,
                 totalWeight: fileSize + 200,
                 headersSize: 200,
                 bodySize: fileSize,
@@ -179,7 +179,7 @@ describe('fileMinifier', function() {
             type: 'css',
             contentLength: 999,
             weightCheck: {
-                body: fileContent.toString('utf8'),
+                bodyBuffer: fileContent,
                 totalWeight: fileSize + 200,
                 headersSize: 200,
                 bodySize: fileSize,

test/core/fontAnalyzerTest.js

@@ -0,0 +1,114 @@
+var should = require('chai').should();
+var fontAnalyzer = require('../../lib/tools/redownload/fontAnalyzer');
+var fs = require('fs');
+var path = require('path');
+
+describe('fontAnalyzer', function() {
+    
+    it('should extract metrics from a font', function(done) {
+        this.timeout(10000);
+
+        var fileContent = fs.readFileSync(path.resolve(__dirname, '../www/SourceSansPro/SourceSansPro-Regular.woff'));
+        var fileSize = fileContent.length;
+
+        var entry = {
+            method: 'GET',
+            url: 'http://localhost:8388/SourceSansPro/SourceSansPro-Regular.woff',
+            requestHeaders: {
+                'User-Agent': 'something',
+                Referer: 'http://www.google.fr/',
+                Accept: '*/*',
+                'Accept-Encoding': 'gzip, deflate'
+            },
+            status: 200,
+            isWebFont: true,
+            type: 'webfont',
+            contentType: 'image/jpeg',
+            contentLength: 999,
+            weightCheck: {
+                bodyBuffer: fileContent,
+                totalWeight: fileSize + 200,
+                headersSize: 200,
+                bodySize: fileSize,
+                isCompressed: false,
+                uncompressedSize: fileSize
+            }
+        };
+
+        fontAnalyzer.getMetricsFromFont(entry, 'ABCD')
+
+        .then(function(metrics) {
+
+            metrics.should.be.an('Object');
+            metrics.should.have.a.property('name').that.equals('Source Sans Pro');
+            metrics.should.have.a.property('numGlyphs').that.equals(1944);
+            metrics.should.have.a.property('averageGlyphComplexity').that.equals(26.6);
+            metrics.should.have.a.property('compressedWeight').that.equals(fileSize);
+
+            metrics.should.have.a.property('unicodeRanges').that.is.an('Object');
+            metrics.unicodeRanges.should.have.a.property('Basic Latin');
+            metrics.unicodeRanges['Basic Latin'].should.have.a.property('charset').that.is.a('String');
+            metrics.unicodeRanges['Basic Latin'].charset.length.should.equal(95);
+            metrics.unicodeRanges['Basic Latin'].name.should.equal('Basic Latin');
+            metrics.unicodeRanges['Basic Latin'].rangeStart.should.equal(0x0020);
+            metrics.unicodeRanges['Basic Latin'].rangeEnd.should.equal(0x007F);
+            metrics.unicodeRanges['Basic Latin'].coverage.should.equal(95 / 96);
+            metrics.unicodeRanges['Basic Latin'].numGlyphsInCommonWithPageContent.should.equal(4);
+
+            metrics.unicodeRanges.Cyrillic.numGlyphsInCommonWithPageContent.should.equal(0);
+
+            metrics.should.have.a.property('numGlyphsInCommonWithPageContent').that.equals(4);
+
+            should.equal(metrics.unicodeRanges.Others.coverage, undefined);
+
+            done();
+        })
+
+        .fail(function(err) {
+            done(err);
+        });
+    });
+
+    it('should sort glyphes by unicode ranges', function() {
+        var ranges = fontAnalyzer.readUnicodeRanges([0x0041, 0x0042, 0x0043, 0x0044, 0x0416], '0123AMZ');
+        
+        ranges.should.deep.equal({ 
+            'Basic Latin': {
+                name: 'Basic Latin',
+                rangeStart: 32,
+                rangeEnd: 127,
+                charset: 'ABCD',
+                coverage: 0.041666666666666664,
+                numGlyphsInCommonWithPageContent: 1
+            },
+            'Cyrillic': {
+                name: 'Cyrillic',
+                rangeStart: 1024,
+                rangeEnd: 1327,
+                charset: 'Ж',
+                coverage: 0.003289473684210526,
+                numGlyphsInCommonWithPageContent: 0
+            }
+        });
+    });
+
+    it('should transform an array of char codes into a string', function() {
+        var str = fontAnalyzer.getCharacterSetAsString([0x0041, 0x0042, 0x0043, 0x0044, 0x0416]);
+        str.should.equal('ABCDЖ');
+    });
+
+    it('should find the right unicode range for a char', function() {
+        fontAnalyzer.getUnicodeRangeFromChar(0x0020).should.deep.equal({
+            name: 'Basic Latin',
+            rangeStart: 0x0020,
+            rangeEnd: 0x007F
+        });
+
+        fontAnalyzer.getUnicodeRangeFromChar(0x0021).name.should.equal('Basic Latin');
+        fontAnalyzer.getUnicodeRangeFromChar(0x007F).name.should.equal('Basic Latin');
+        fontAnalyzer.getUnicodeRangeFromChar(0x007E).name.should.equal('Basic Latin');
+        fontAnalyzer.getUnicodeRangeFromChar(0x0000).name.should.equal('Others');
+        fontAnalyzer.getUnicodeRangeFromChar(0xFFFFFFFFF).name.should.equal('Others');
+    });
+
+});

test/core/gzipCompressorTest.js

@@ -1,6 +1,6 @@
 var should = require('chai').should();
-var gzipCompressor = require('../../lib/tools/weightChecker/gzipCompressor');
-var fileMinifier = require('../../lib/tools/weightChecker/fileMinifier');
+var gzipCompressor = require('../../lib/tools/redownload/gzipCompressor');
+var fileMinifier = require('../../lib/tools/redownload/fileMinifier');
 var fs = require('fs');
 var path = require('path');
 
@@ -23,7 +23,7 @@ describe('gzipCompressor', function() {
             type: 'js',
             contentLength: 999,
             weightCheck: {
-                body: fileContent.toString('utf8'),
+                bodyBuffer: fileContent,
                 totalWeight: fileSize + 200,
                 headersSize: 200,
                 bodySize: fileSize,
@@ -64,7 +64,7 @@ describe('gzipCompressor', function() {
             type: 'js',
             contentLength: 999,
             weightCheck: {
-                body: fileContent.toString('utf8'),
+                bodyBuffer: fileContent,
                 bodyAfterOptimization: minifiedContent.toString('utf8'),
                 totalWeight: fileSize + 200,
                 headersSize: 200,
@@ -108,7 +108,7 @@ describe('gzipCompressor', function() {
             type: 'js',
             contentLength: 999,
             weightCheck: {
-                body: fileContent.toString('utf8'),
+                bodyBuffer: fileContent,
                 bodyAfterOptimization: minifiedContent.toString('utf8'),
                 totalWeight: gzipedSize + 200,
                 headersSize: 200,
@@ -150,7 +150,7 @@ describe('gzipCompressor', function() {
             type: 'js',
             contentLength: 999,
             weightCheck: {
-                body: fileContent.toString('utf8'),
+                bodyBuffer: fileContent,
                 totalWeight: gzipedSize + 200,
                 headersSize: 200,
                 bodySize: gzipedSize,
@@ -188,7 +188,7 @@ describe('gzipCompressor', function() {
             type: 'css',
             contentLength: 999,
             weightCheck: {
-                body: fileContent.toString('utf8'),
+                bodyBuffer: fileContent,
                 totalWeight: fileSize + 200,
                 headersSize: 200,
                 bodySize: fileSize,
@@ -224,7 +224,7 @@ describe('gzipCompressor', function() {
             type: 'html',
             contentLength: 999,
             weightCheck: {
-                body: fileContent.toString('utf8'),
+                bodyBuffer: fileContent,
                 totalWeight: fileSize + 200,
                 headersSize: 200,
                 bodySize: fileSize,
@@ -261,7 +261,7 @@ describe('gzipCompressor', function() {
             type: 'image',
             contentLength: 999,
             weightCheck: {
-                body: fileContent.toString('utf8'),
+                bodyBuffer: fileContent,
                 totalWeight: fileSize + 200,
                 headersSize: 200,
                 bodySize: fileSize,
@@ -297,7 +297,7 @@ describe('gzipCompressor', function() {
             type: 'xml',
             contentLength: 999,
             weightCheck: {
-                body: fileContent.toString('utf8'),
+                bodyBuffer: fileContent,
                 totalWeight: fileSize + 200,
                 headersSize: 200,
                 bodySize: fileSize,
@@ -333,7 +333,7 @@ describe('gzipCompressor', function() {
             type: 'json',
             contentLength: 999,
             weightCheck: {
-                body: fileContent.toString('utf8'),
+                bodyBuffer: fileContent,
                 totalWeight: fileSize + 200,
                 headersSize: 200,
                 bodySize: fileSize,
@@ -370,7 +370,7 @@ describe('gzipCompressor', function() {
             type: 'webfont',
             contentLength: 999,
             weightCheck: {
-                body: fileContent.toString('utf8'),
+                bodyBuffer: fileContent,
                 totalWeight: fileSize + 200,
                 headersSize: 200,
                 bodySize: fileSize,
@@ -407,7 +407,7 @@ describe('gzipCompressor', function() {
             type: 'favicon',
             contentLength: 999,
             weightCheck: {
-                body: fileContent.toString('utf8'),
+                bodyBuffer: fileContent,
                 totalWeight: fileSize + 200,
                 headersSize: 200,
                 bodySize: fileSize,
@@ -444,7 +444,7 @@ describe('gzipCompressor', function() {
             contentType: 'image/jpeg',
             contentLength: 999,
             weightCheck: {
-                body: fileContent.toString('utf8'),
+                bodyBuffer: fileContent,
                 totalWeight: fileSize + 200,
                 headersSize: 200,
                 bodySize: fileSize,
@@ -482,7 +482,7 @@ describe('gzipCompressor', function() {
             contentType: 'image/png',
             contentLength: 999,
             weightCheck: {
-                body: fileContent.toString('utf8'),
+                bodyBuffer: fileContent,
                 totalWeight: fileSize + 200,
                 headersSize: 200,
                 bodySize: fileSize,
@@ -519,7 +519,7 @@ describe('gzipCompressor', function() {
             contentType: 'image/gif',
             contentLength: 999,
             weightCheck: {
-                body: fileContent.toString('utf8'),
+                bodyBuffer: fileContent,
                 totalWeight: fileSize + 200,
                 headersSize: 200,
                 bodySize: fileSize,
@@ -556,7 +556,7 @@ describe('gzipCompressor', function() {
             contentType: 'image/webp',
             contentLength: 999,
             weightCheck: {
-                body: fileContent.toString('utf8'),
+                bodyBuffer: fileContent,
                 totalWeight: fileSize + 200,
                 headersSize: 200,
                 bodySize: fileSize,

test/core/imageOptimizerTest.js

@@ -1,5 +1,5 @@
 var should = require('chai').should();
-var imageOptimizer = require('../../lib/tools/weightChecker/imageOptimizer');
+var imageOptimizer = require('../../lib/tools/redownload/imageOptimizer');
 var fs = require('fs');
 var path = require('path');
 
@@ -11,7 +11,7 @@ describe('imageOptimizer', function() {
         var fileSize = fileContent.length;
 
         imageOptimizer.compressJpegLosslessly(fileContent).then(function(newFile) {
-            var newFileSize = newFile.contents.length;
+            var newFileSize = newFile.length;
             newFileSize.should.be.below(fileSize);
             done();
         }).fail(function(err) {
@@ -25,7 +25,7 @@ describe('imageOptimizer', function() {
         var fileSize = fileContent.length;
 
         imageOptimizer.compressJpegLossly(fileContent).then(function(newFile) {
-            var newFileSize = newFile.contents.length;
+            var newFileSize = newFile.length;
             newFileSize.should.be.below(fileSize);
             done();
         }).fail(function(err) {
@@ -52,7 +52,7 @@ describe('imageOptimizer', function() {
             contentType: 'image/jpeg',
             contentLength: 999,
             weightCheck: {
-                body: fileContent,
+                bodyBuffer: fileContent,
                 totalWeight: fileSize + 200,
                 headersSize: 200,
                 bodySize: fileSize,
@@ -82,7 +82,7 @@ describe('imageOptimizer', function() {
         var fileSize = fileContent.length;
 
         imageOptimizer.compressPngLosslessly(fileContent).then(function(newFile) {
-            var newFileSize = newFile.contents.length;
+            var newFileSize = newFile.length;
             newFileSize.should.be.below(fileSize);
             done();
         }).fail(function(err) {
@@ -96,7 +96,7 @@ describe('imageOptimizer', function() {
         var fileSize = fileContent.length;
 
         imageOptimizer.compressPngLosslessly(fileContent).then(function(newFile) {
-            var newFileSize = newFile.contents.length;
+            var newFileSize = newFile.length;
             newFileSize.should.equal(fileSize);
             done();
         }).fail(function(err) {
@@ -110,7 +110,7 @@ describe('imageOptimizer', function() {
         var fileSize = fileContent.length;
 
         imageOptimizer.compressPngLosslessly(fileContent).then(function(newFile) {
-            var newFileSize = newFile.contents.length;
+            var newFileSize = newFile.length;
             newFileSize.should.equal(fileSize);
             done();
         }).fail(function(err) {
@@ -137,7 +137,7 @@ describe('imageOptimizer', function() {
             contentType: 'image/png',
             contentLength: 999,
             weightCheck: {
-                body: fileContent,
+                bodyBuffer: fileContent,
                 totalWeight: fileSize + 200,
                 headersSize: 200,
                 bodySize: fileSize,
@@ -166,7 +166,7 @@ describe('imageOptimizer', function() {
         var fileSize = fileContent.length;
 
         imageOptimizer.compressSvgLosslessly(fileContent).then(function(newFile) {
-            var newFileSize = newFile.contents.length;
+            var newFileSize = newFile.length;
             newFileSize.should.be.below(fileSize);
             done();
         }).fail(function(err) {
@@ -194,7 +194,7 @@ describe('imageOptimizer', function() {
             contentType: 'image/svg+xml',
             contentLength: 999,
             weightCheck: {
-                body: fileContent,
+                bodyBuffer: fileContent,
                 totalWeight: fileSize + 200,
                 headersSize: 200,
                 bodySize: fileSize,
@@ -242,7 +242,7 @@ describe('imageOptimizer', function() {
             contentType: 'image/jpeg',
             contentLength: 999,
             weightCheck: {
-                body: fileContent,
+                bodyBuffer: fileContent,
                 totalWeight: fileSize + 200,
                 headersSize: 200,
                 bodySize: fileSize,
@@ -285,7 +285,7 @@ describe('imageOptimizer', function() {
             contentType: 'image/png',
             contentLength: 999,
             weightCheck: {
-                body: fileContent,
+                bodyBuffer: fileContent,
                 totalWeight: fileSize + 200,
                 headersSize: 200,
                 bodySize: fileSize,

test/core/indexTest.js

@@ -66,9 +66,9 @@ describe('index.js', function() {
                         "tool": "phantomas",
                         "label": "DOM max depth",
                         "message": "<p>A deep DOM makes the CSS matching with DOM elements difficult.</p><p>It also slows down JavaScript modifications to the DOM because changing the dimensions of an element makes the browser re-calculate the dimensions of it's parents. Same thing for JavaScript events, that bubble up to the document root.</p>",
-                        "isOkThreshold": 10,
-                        "isBadThreshold": 20,
-                        "isAbnormalThreshold": 28,
+                        "isOkThreshold": 12,
+                        "isBadThreshold": 22,
+                        "isAbnormalThreshold": 30,
                         "hasOffenders": true
                     },
                     "value": 1,
@@ -109,6 +109,21 @@ describe('index.js', function() {
             });
     });
 
+    it('should succeed on try-catch.html', function(done) {
+        this.timeout(15000);
+
+        var url = 'http://localhost:8388/try-catch.html';
+
+        ylt(url)
+            .then(function(data) {
+                data.toolsResults.phantomas.metrics.should.have.a.property('jsErrors').that.equals(0);
+                done();
+            }).fail(function(err) {
+                console.log.restore();
+                done(err);
+            });
+    });
+
     it('should take a screenshot', function(done) {
         this.timeout(15000);
 

test/core/isHttp2Test.js

@@ -0,0 +1,48 @@
+var should = require('chai').should();
+var isHttp2 = require('../../lib/tools/isHttp2');
+
+describe('isHttp2', function() {
+    
+    it('should parse the protocol correctly', function() {
+        isHttp2.getProtocol({
+            toolsResults: {
+                phantomas: {
+                    url: 'http://www.yahoo.com'
+                }
+            }
+        }).should.equal('http:');
+
+
+        isHttp2.getProtocol({
+            toolsResults: {
+                phantomas: {
+                    url: 'https://www.yahoo.com'
+                }
+            }
+        }).should.equal('https:');
+    });
+
+    it('should parse the domain correctly', function() {
+        isHttp2.getDomain({
+            toolsResults: {
+                phantomas: {
+                    url: 'http://www.yahoo.com'
+                }
+            }
+        }).should.equal('www.yahoo.com');
+
+
+        isHttp2.getDomain({
+            toolsResults: {
+                phantomas: {
+                    url: 'https://www.yahoo.com'
+                }
+            }
+        }).should.equal('www.yahoo.com');
+    });
+
+    it('should have a function checkHttp2', function() {
+        isHttp2.should.have.a.property('checkHttp2').that.is.a('function');
+    });
+
+});

test/core/mediaQueriesCheckerTest.js

@@ -0,0 +1,55 @@
+var should = require('chai').should();
+var mediaQueriesChecker = require('../../lib/tools/mediaQueriesChecker');
+
+describe('mediaQueriesChecker', function() {
+    
+    it('should parse mediaQueryes correctly', function() {
+        mediaQueriesChecker.parseOneMediaQuery('@media screen and (max-width: 1024px) (1 rules) <http://domain.com/css/stylesheet.css> @ 269:1').should.deep.equal({
+            mediaQuery: {
+                query: 'screen and (max-width: 1024px)',
+                rules: 1,
+                file: 'http://domain.com/css/stylesheet.css',
+                line: 269,
+                column: 1
+            },
+            isForMobile: true,
+            breakpoints: [{string: '1024px', pixels: 1024}]
+        });
+    });
+
+    it('should determine if it\'s for mobile correctly', function() {
+        mediaQueriesChecker.parseOneMediaQuery('@media screen and (max-width: 1024px) (1 rules) <file> @ 1:1').isForMobile.should.equal(true);
+        mediaQueriesChecker.parseOneMediaQuery('@media (max-width:1024px) (1 rules) <file> @ 1:1').isForMobile.should.equal(true);
+        mediaQueriesChecker.parseOneMediaQuery('@media screen and (max-width: 320px) (1 rules) <file> @ 1:1').isForMobile.should.equal(true);
+        mediaQueriesChecker.parseOneMediaQuery('@media screen and (max-width: 290px) (1 rules) <file> @ 1:1').isForMobile.should.equal(false);
+        mediaQueriesChecker.parseOneMediaQuery('@media screen and (min-width: 320px) (1 rules) <file> @ 1:1').isForMobile.should.equal(false);
+        mediaQueriesChecker.parseOneMediaQuery('@media screen and (min-width: 600px) (1 rules) <file> @ 1:1').isForMobile.should.equal(false);
+        mediaQueriesChecker.parseOneMediaQuery('@media screen and (max-width: 20em) (1 rules) <file> @ 1:1').isForMobile.should.equal(true);
+        mediaQueriesChecker.parseOneMediaQuery('@media screen and (min-width: 40em) (1 rules) <file> @ 1:1').isForMobile.should.equal(false);
+        mediaQueriesChecker.parseOneMediaQuery('@media (min-width: 600px) and (max-width: 1000px) (1 rules) <file> @ 1:1').isForMobile.should.equal(false);
+        mediaQueriesChecker.parseOneMediaQuery('@media (min-width: 180px) and (max-width: 400px) (1 rules) <file> @ 1:1').isForMobile.should.equal(true);
+        mediaQueriesChecker.parseOneMediaQuery('@media (min-width: 180px) and (max-width: 290px) (1 rules) <file> @ 1:1').isForMobile.should.equal(false);
+        mediaQueriesChecker.parseOneMediaQuery('@media not all and (min-width: 600px) (1 rules) <file> @ 1:1').isForMobile.should.equal(true);
+        mediaQueriesChecker.parseOneMediaQuery('@media not all and (min-width: 180px) (1 rules) <file> @ 1:1').isForMobile.should.equal(false);
+        mediaQueriesChecker.parseOneMediaQuery('@media not all and (min-width: 1000px) and (max-width: 600px) (1 rules) <file> @ 1:1').isForMobile.should.equal(false);
+        mediaQueriesChecker.parseOneMediaQuery('@media not all and (min-width: 400px) and (max-width: 180px) (1 rules) <file> @ 1:1').isForMobile.should.equal(true);
+    });
+
+    it('should count breakpoints correctly', function() {
+        mediaQueriesChecker.parseOneMediaQuery('@media screen and (max-width: 1024px) (1 rules) <file> @ 1:1').breakpoints.should.deep.equal([{string: '1024px', pixels: 1024}]);
+        mediaQueriesChecker.parseOneMediaQuery('@media (max-width:1024px) (1 rules) <file> @ 1:1').breakpoints.should.deep.equal([{string: '1024px', pixels: 1024}]);
+        mediaQueriesChecker.parseOneMediaQuery('@media screen and (max-width: 320px) (1 rules) <file> @ 1:1').breakpoints.should.deep.equal([{string: '320px', pixels: 320}]);
+        mediaQueriesChecker.parseOneMediaQuery('@media (min-width: 600px) and (max-width: 1000px) (1 rules) <file> @ 1:1').breakpoints.should.deep.equal([{string: '600px', pixels: 600}, {string: '1000px', pixels: 1000}]);
+        mediaQueriesChecker.parseOneMediaQuery('@media not all and (min-width: 180px) (1 rules) <file> @ 1:1').breakpoints.should.deep.equal([{string: '180px', pixels: 180}]);
+        mediaQueriesChecker.parseOneMediaQuery('@media not all and (min-width: 1000px) and (max-width: 600px) (1 rules) <file> @ 1:1').breakpoints.should.deep.equal([{string: '1000px', pixels: 1000}, {string: '600px', pixels: 600}]);
+        mediaQueriesChecker.parseOneMediaQuery('@media (max-height:500px) (1 rules) <file> @ 1:1').breakpoints.should.deep.equal([]);
+        mediaQueriesChecker.parseOneMediaQuery('@media screen and (max-width: 100em) (1 rules) <file> @ 1:1').breakpoints.should.deep.equal([{string: '100em', pixels: 1600}]);
+        mediaQueriesChecker.parseOneMediaQuery('@media screen and (max-width: 120pt) (1 rules) <file> @ 1:1').breakpoints.should.deep.equal([{string: '120pt', pixels: 160}]);
+        mediaQueriesChecker.parseOneMediaQuery('@media screen and (max-width: 40.2em) (1 rules) <file> @ 1:1').breakpoints.should.deep.equal([{string: '40.2em', pixels: 643.2}]);
+    });
+
+    it('should fail silently', function() {
+        mediaQueriesChecker.parseOneMediaQuery('@media bad syntax (1 rules) <file> @ 1:1').isForMobile.should.equal(false); 
+    });
+
+});

test/core/offendersHelpersTest.js

@@ -200,6 +200,47 @@ describe('offendersHelpers', function() {
             ]);
         });
 
+        it('should transform a backtrace with the new PhantomJS 2.x syntax into an array', function() {
+            var result = offendersHelpers.backtraceToArray('each@http://m.australia.fr/js/min/vendors.js?20160706185900:4:5365 / f@http://m.australia.fr/js/min/vendors.js?20160706185900:17:82 / http://m.australia.fr/js/min/vendors.js?20160706185900:17:855 / handle@http://m.australia.fr/js/min/vendors.js?20160706185900:5:10871 / report@phantomjs://platform/phantomas.js:535:20 / phantomjs://platform/phantomas.js:524:15');
+
+            result.should.deep.equal([
+                {
+                    functionName: 'each',
+                    file: 'http://m.australia.fr/js/min/vendors.js?20160706185900',
+                    line: 4,
+                    column: 5365
+                },
+                {
+                    functionName: 'f',
+                    file: 'http://m.australia.fr/js/min/vendors.js?20160706185900',
+                    line: 17,
+                    column: 82
+                },
+                {
+                    file: 'http://m.australia.fr/js/min/vendors.js?20160706185900',
+                    line: 17,
+                    column: 855
+                },
+                {
+                    functionName: 'handle',
+                    file: 'http://m.australia.fr/js/min/vendors.js?20160706185900',
+                    line: 5,
+                    column: 10871
+                },
+                {
+                    functionName: 'report',
+                    file: 'phantomjs://platform/phantomas.js',
+                    line: 535,
+                    column: 20
+                },
+                {
+                    file: 'phantomjs://platform/phantomas.js',
+                    line: 524,
+                    column: 15
+                }
+            ]);
+        });
+
         it('should return null if it fails', function() {
             var result = offendersHelpers.backtraceToArray('http://pouet.com/js/jquery.footer-transverse-min-v1.0.20.js:1 /http://pouet.com/js/main.js:1');
 
@@ -299,6 +340,17 @@ describe('offendersHelpers', function() {
             });
         });
 
+        it('should handle line breaks inside the string', function() {
+            var result = offendersHelpers.cssOffenderPattern('.card-mask-wrap { -moz-transform: translate3d(0, \n-288px\n, 0) } // was required by firefox 15 and earlier [inline CSS] @ 29:3');
+
+            result.should.deep.equal({
+                css: '.card-mask-wrap { -moz-transform: translate3d(0, -288px, 0) } // was required by firefox 15 and earlier',
+                file: null,
+                line: 29,
+                column: 3
+            });
+        }); 
+
     });
 
     describe('fileWithSizePattern', function() {

test/core/phantomasWrapperTest.js

@@ -51,11 +51,14 @@ describe('phantomasWrapper', function() {
             done('Error: unwanted success');
 
         }).fail(function(err) {
+            try {
+                should.exist(err);
+                err.should.equal(254);
 
-            should.exist(err);
-            err.should.equal(254);
-
-            done();
+                done();
+            } catch(error) {
+                done(error);
+            }
         });
     });
 
@@ -72,16 +75,20 @@ describe('phantomasWrapper', function() {
             }
         }).then(function(data) {
             /*jshint -W030 */
-            
-            data.should.be.an('object');
-            data.should.have.a.property('generator');
-            data.generator.should.contain('phantomas');
-            data.should.have.a.property('url').that.equals(url);
-            data.should.have.a.property('metrics').that.is.an('object').not.empty;
-            data.should.have.a.property('offenders').that.is.an('object').not.empty;
-            data.offenders.should.have.a.property('javascriptExecutionTree').that.is.a('array').not.empty;
 
-            done();
+            try {            
+                data.should.be.an('object');
+                data.should.have.a.property('generator');
+                data.generator.should.contain('phantomas');
+                data.should.have.a.property('url').that.equals(url);
+                data.should.have.a.property('metrics').that.is.an('object').not.empty;
+                data.should.have.a.property('offenders').that.is.an('object').not.empty;
+                data.offenders.should.have.a.property('javascriptExecutionTree').that.is.a('array').not.empty;
+
+                done();
+            } catch(error) {
+                done(error);
+            }
         }).fail(function(err) {
             done(err);
         });

test/core/redownloadTest.js

@@ -1,9 +1,9 @@
 var should = require('chai').should();
-var weightChecker = require('../../lib/tools/weightChecker/weightChecker');
+var redownload = require('../../lib/tools/redownload/redownload');
 var fs = require('fs');
 var path = require('path');
 
-describe('weightChecker', function() {
+describe('redownload', function() {
     
     it('should download a list of files', function(done) {
         this.timeout(10000);
@@ -84,6 +84,18 @@ describe('weightChecker', function() {
                 isCSS: true,
                 type: 'css'
             },
+            {
+                method: 'GET',
+                url: 'http://localhost:8388/xml.xml',
+                requestHeaders: {
+                    'User-Agent': 'something',
+                   Referer: 'http://www.google.fr/',
+                   Accept: '*/*'
+                },
+                status: 200,
+                isXML: true,
+                type: 'xml'
+            },
             {
                 method: 'GET',
                 url: 'about:blank',
@@ -109,49 +121,49 @@ describe('weightChecker', function() {
             }
         };
 
-        weightChecker.recheckAllFiles(data)
+        redownload.recheckAllFiles(data)
 
         .then(function(data) {
-            data.toolsResults.should.have.a.property('weightChecker');
-            data.toolsResults.weightChecker.should.have.a.property('metrics');
-            data.toolsResults.weightChecker.should.have.a.property('offenders');
+            data.toolsResults.should.have.a.property('redownload');
+            data.toolsResults.redownload.should.have.a.property('metrics');
+            data.toolsResults.redownload.should.have.a.property('offenders');
 
-            data.toolsResults.weightChecker.offenders.should.have.a.property('totalWeight');
-            data.toolsResults.weightChecker.offenders.totalWeight.totalWeight.should.be.above(0);
-            data.toolsResults.weightChecker.offenders.totalWeight.byType.html.requests.length.should.equal(1);
-            data.toolsResults.weightChecker.offenders.totalWeight.byType.js.requests.length.should.equal(2);
-            data.toolsResults.weightChecker.offenders.totalWeight.byType.css.requests.length.should.equal(1);
-            data.toolsResults.weightChecker.offenders.totalWeight.byType.image.requests.length.should.equal(2);
-            data.toolsResults.weightChecker.offenders.totalWeight.byType.other.requests.length.should.equal(0);
+            data.toolsResults.redownload.offenders.should.have.a.property('totalWeight');
+            data.toolsResults.redownload.offenders.totalWeight.totalWeight.should.be.above(0);
+            data.toolsResults.redownload.offenders.totalWeight.byType.html.requests.length.should.equal(1);
+            data.toolsResults.redownload.offenders.totalWeight.byType.js.requests.length.should.equal(2);
+            data.toolsResults.redownload.offenders.totalWeight.byType.css.requests.length.should.equal(1);
+            data.toolsResults.redownload.offenders.totalWeight.byType.image.requests.length.should.equal(2);
+            data.toolsResults.redownload.offenders.totalWeight.byType.other.requests.length.should.equal(1);
 
-            data.toolsResults.weightChecker.offenders.should.have.a.property('imageOptimization');
-            data.toolsResults.weightChecker.offenders.imageOptimization.totalGain.should.be.above(0);
-            data.toolsResults.weightChecker.offenders.imageOptimization.images.length.should.equal(2);
+            data.toolsResults.redownload.offenders.should.have.a.property('imageOptimization');
+            data.toolsResults.redownload.offenders.imageOptimization.totalGain.should.be.above(0);
+            data.toolsResults.redownload.offenders.imageOptimization.images.length.should.equal(2);
 
-            data.toolsResults.weightChecker.offenders.should.have.a.property('gzipCompression');
-            data.toolsResults.weightChecker.offenders.gzipCompression.totalGain.should.be.above(0);
-            data.toolsResults.weightChecker.offenders.gzipCompression.files.length.should.equal(4);
+            data.toolsResults.redownload.offenders.should.have.a.property('gzipCompression');
+            data.toolsResults.redownload.offenders.gzipCompression.totalGain.should.be.above(0);
+            data.toolsResults.redownload.offenders.gzipCompression.files.length.should.equal(5);
 
-            data.toolsResults.weightChecker.offenders.should.have.a.property('fileMinification');
-            data.toolsResults.weightChecker.offenders.fileMinification.totalGain.should.be.above(0);
-            data.toolsResults.weightChecker.offenders.fileMinification.files.length.should.equal(2);
+            data.toolsResults.redownload.offenders.should.have.a.property('fileMinification');
+            data.toolsResults.redownload.offenders.fileMinification.totalGain.should.be.above(0);
+            data.toolsResults.redownload.offenders.fileMinification.files.length.should.equal(2);
 
-            data.toolsResults.weightChecker.metrics.should.have.a.property('totalRequests').that.equals(6);
-            data.toolsResults.weightChecker.offenders.should.have.a.property('totalRequests');
-            data.toolsResults.weightChecker.offenders.totalRequests.byType.html.length.should.equal(1);
-            data.toolsResults.weightChecker.offenders.totalRequests.byType.js.length.should.equal(2);
-            data.toolsResults.weightChecker.offenders.totalRequests.byType.css.length.should.equal(1);
-            data.toolsResults.weightChecker.offenders.totalRequests.byType.image.length.should.equal(2);
-            data.toolsResults.weightChecker.offenders.totalRequests.byType.json.length.should.equal(0);
-            data.toolsResults.weightChecker.offenders.totalRequests.byType.webfont.length.should.equal(0);
-            data.toolsResults.weightChecker.offenders.totalRequests.byType.video.length.should.equal(0);
-            data.toolsResults.weightChecker.offenders.totalRequests.byType.other.length.should.equal(0);
+            data.toolsResults.redownload.metrics.should.have.a.property('totalRequests').that.equals(7);
+            data.toolsResults.redownload.offenders.should.have.a.property('totalRequests');
+            data.toolsResults.redownload.offenders.totalRequests.byType.html.length.should.equal(1);
+            data.toolsResults.redownload.offenders.totalRequests.byType.js.length.should.equal(2);
+            data.toolsResults.redownload.offenders.totalRequests.byType.css.length.should.equal(1);
+            data.toolsResults.redownload.offenders.totalRequests.byType.image.length.should.equal(2);
+            data.toolsResults.redownload.offenders.totalRequests.byType.json.length.should.equal(0);
+            data.toolsResults.redownload.offenders.totalRequests.byType.webfont.length.should.equal(0);
+            data.toolsResults.redownload.offenders.totalRequests.byType.video.length.should.equal(0);
+            data.toolsResults.redownload.offenders.totalRequests.byType.other.length.should.equal(1);
 
-            data.toolsResults.weightChecker.metrics.should.have.a.property('smallRequests').that.equals(0);
-            data.toolsResults.weightChecker.offenders.should.have.a.property('smallRequests');
-            data.toolsResults.weightChecker.offenders.smallRequests.byType.js.length.should.equal(0);
-            data.toolsResults.weightChecker.offenders.smallRequests.byType.css.length.should.equal(0);
-            data.toolsResults.weightChecker.offenders.smallRequests.byType.image.length.should.equal(0);
+            data.toolsResults.redownload.metrics.should.have.a.property('smallRequests').that.equals(0);
+            data.toolsResults.redownload.offenders.should.have.a.property('smallRequests');
+            data.toolsResults.redownload.offenders.smallRequests.byType.js.length.should.equal(0);
+            data.toolsResults.redownload.offenders.smallRequests.byType.css.length.should.equal(0);
+            data.toolsResults.redownload.offenders.smallRequests.byType.image.length.should.equal(0);
 
             done();
         })
@@ -175,7 +187,7 @@ describe('weightChecker', function() {
             type: 'js'
         };
 
-        weightChecker.redownloadEntry(entry)
+        redownload.redownloadEntry(entry)
 
         .then(function(newEntry) {
 
@@ -183,7 +195,7 @@ describe('weightChecker', function() {
             newEntry.weightCheck.uncompressedSize.should.equal(newEntry.weightCheck.bodySize);
             newEntry.weightCheck.isCompressed.should.equal(false);
             newEntry.weightCheck.headersSize.should.be.above(200).and.below(400);
-            newEntry.weightCheck.body.toString().should.have.string('1.8.3');
+            newEntry.weightCheck.bodyBuffer.toString().should.have.string('1.8.3');
 
             done();
         })
@@ -209,22 +221,21 @@ describe('weightChecker', function() {
             contentType: 'image/png'
         };
 
-        weightChecker.redownloadEntry(entry)
+        redownload.redownloadEntry(entry)
 
         .then(function(newEntry) {
 
             newEntry.weightCheck.bodySize.should.equal(4193);
-            newEntry.weightCheck.body.should.equal(fileContent.toString('binary'));
+            newEntry.weightCheck.bodyBuffer.should.deep.equal(fileContent);
 
-            // Opening the image in lwip to check if the format is good
-            var lwip = require('lwip');
-            var buffer = new Buffer(newEntry.weightCheck.body, 'binary');
-            lwip.open(buffer, 'png', function(err, image) {
-                image.width().should.equal(620);
-                image.height().should.equal(104);
+            // Opening the image in jimp to check if the format is good
+            var Jimp = require('jimp');
+            Jimp.read(newEntry.weightCheck.bodyBuffer, function(err, image) {
+                image.bitmap.width.should.equal(620);
+                image.bitmap.height.should.equal(104);
                 done(err);
             });
-            
+
         })
 
         .fail(function(err) {
@@ -246,7 +257,7 @@ describe('weightChecker', function() {
             contentLength: 999
         };
 
-        weightChecker.redownloadEntry(entry)
+        redownload.redownloadEntry(entry)
 
         .then(function(newEntry) {
             newEntry.weightCheck.should.have.a.property('message').that.equals('error while downloading: 404');
@@ -273,7 +284,7 @@ describe('weightChecker', function() {
             contentLength: 999
         };
 
-        weightChecker.redownloadEntry(entry)
+        redownload.redownloadEntry(entry)
 
         .then(function(newEntry) {
             newEntry.should.not.have.a.property('weightCheck');
@@ -286,7 +297,7 @@ describe('weightChecker', function() {
     });
 
     it('should listRequestWeight', function() {
-        var totalWeightObj = weightChecker.listRequestWeight([{
+        var totalWeightObj = redownload.listRequestWeight([{
             method: 'GET',
             url: 'http://localhost:8388/jquery1.8.3.js',
             requestHeaders: {
@@ -300,7 +311,7 @@ describe('weightChecker', function() {
             type: 'html',
             contentLength: 999,
             weightCheck: {
-                body: 'blabla',
+                bodyBuffer: 'blabla',
                 headersSize: 200,
                 bodySize: 500,
                 isCompressed: false,
@@ -316,7 +327,7 @@ describe('weightChecker', function() {
     });
 
     it('should listRequestWeight even if download failed', function() {
-        var totalWeightObj = weightChecker.listRequestWeight([{
+        var totalWeightObj = redownload.listRequestWeight([{
             method: 'GET',
             url: 'http://localhost:8388/jquery1.8.3.js',
             requestHeaders: {

test/fixtures/settings.json

@@ -1,10 +1,15 @@
 {
     "serverPort": "8387",
+    "phantomasEngine": "webkit",
     "googleAnalyticsId": "",
-    
+    "screenshotWidth": 400,
+    "baseUrl": "/",    
     "authorizedKeys": {
         "1234567890": "contact@gaelmetais.com"
     },
     "maxAnonymousRunsPerDay": 10,
-    "maxAnonymousCallsPerDay": 1000
+    "maxAnonymousCallsPerDay": 1000,
+    "blockedUrls": [
+        "http://www.test.com"
+    ]
 }

test/www/SourceSansPro/OFL.txt

@@ -0,0 +1,93 @@
+Copyright 2010, 2012 Adobe Systems Incorporated (http://www.adobe.com/), with Reserved Font Name 'Source'. All Rights Reserved. Source is a trademark of Adobe Systems Incorporated in the United States and/or other countries.
+
+This Font Software is licensed under the SIL Open Font License, Version 1.1.
+
+This license is copied below, and is also available with a FAQ at: http://scripts.sil.org/OFL
+
+
+-----------------------------------------------------------
+SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007
+-----------------------------------------------------------
+
+PREAMBLE
+The goals of the Open Font License (OFL) are to stimulate worldwide
+development of collaborative font projects, to support the font creation
+efforts of academic and linguistic communities, and to provide a free and
+open framework in which fonts may be shared and improved in partnership
+with others.
+
+The OFL allows the licensed fonts to be used, studied, modified and
+redistributed freely as long as they are not sold by themselves. The
+fonts, including any derivative works, can be bundled, embedded, 
+redistributed and/or sold with any software provided that any reserved
+names are not used by derivative works. The fonts and derivatives,
+however, cannot be released under any other type of license. The
+requirement for fonts to remain under this license does not apply
+to any document created using the fonts or their derivatives.
+
+DEFINITIONS
+"Font Software" refers to the set of files released by the Copyright
+Holder(s) under this license and clearly marked as such. This may
+include source files, build scripts and documentation.
+
+"Reserved Font Name" refers to any names specified as such after the
+copyright statement(s).
+
+"Original Version" refers to the collection of Font Software components as
+distributed by the Copyright Holder(s).
+
+"Modified Version" refers to any derivative made by adding to, deleting,
+or substituting -- in part or in whole -- any of the components of the
+Original Version, by changing formats or by porting the Font Software to a
+new environment.
+
+"Author" refers to any designer, engineer, programmer, technical
+writer or other person who contributed to the Font Software.
+
+PERMISSION & CONDITIONS
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of the Font Software, to use, study, copy, merge, embed, modify,
+redistribute, and sell modified and unmodified copies of the Font
+Software, subject to the following conditions:
+
+1) Neither the Font Software nor any of its individual components,
+in Original or Modified Versions, may be sold by itself.
+
+2) Original or Modified Versions of the Font Software may be bundled,
+redistributed and/or sold with any software, provided that each copy
+contains the above copyright notice and this license. These can be
+included either as stand-alone text files, human-readable headers or
+in the appropriate machine-readable metadata fields within text or
+binary files as long as those fields can be easily viewed by the user.
+
+3) No Modified Version of the Font Software may use the Reserved Font
+Name(s) unless explicit written permission is granted by the corresponding
+Copyright Holder. This restriction only applies to the primary font name as
+presented to the users.
+
+4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font
+Software shall not be used to promote, endorse or advertise any
+Modified Version, except to acknowledge the contribution(s) of the
+Copyright Holder(s) and the Author(s) or with their explicit written
+permission.
+
+5) The Font Software, modified or unmodified, in part or in whole,
+must be distributed entirely under this license, and must not be
+distributed under any other license. The requirement for fonts to
+remain under this license does not apply to any document created
+using the Font Software.
+
+TERMINATION
+This license becomes null and void if any of the above conditions are
+not met.
+
+DISCLAIMER
+THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
+OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE
+COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL
+DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM
+OTHER DEALINGS IN THE FONT SOFTWARE.

test/www/font-page.html

@@ -0,0 +1,23 @@
+<html>
+<head>
+<style>
+    body {
+        font-size: 4em;
+    }
+    script, style {
+        display: block;
+        background: #F66;
+    }
+    @font-face {
+        font-family: "SourceSansPro";
+        src: url("SourceSansPro/SourceSansPro-Regular.woff");
+    }
+    .foo {
+        font-family: SourceSansPro;
+    }
+</style>
+</head>
+<body>
+<div class="foo">Some text</div>
+</body>
+</html>

test/www/jquery-page.html

@@ -190,6 +190,11 @@
         $li.parentsUntil('body', 'div');
         $li.siblings();
         $li.siblings('li');
+
+        $.ajax({
+            url: 'xml.xml',
+            async: false
+        });
     </script>
 </body>
 </html>

test/www/try-catch.html

@@ -0,0 +1,21 @@
+<html>
+    <head>
+        <title>Testing getElementById with a try catch statement</title>
+    </head>
+    <body>
+        <div id="foo">Some text</div>
+        <script>
+            try {
+                document.getElementById(undefined);
+                document.getElementsByClassName(undefined);
+                document.getElementsByTagName(undefined);
+                
+            } catch(err) {
+                console.log('Error found: ' + err);
+                throw new Error("I detect an error!");
+            }
+
+            document.getElementById('foo');
+        </script>
+    </body>
+</html>

test/www/xml.xml

@@ -0,0 +1,80 @@
+<?xml version="1.0"?>
+<catalog>
+   <book id="bk101">
+      <author>Gambardella, Matthew</author>
+      <title>XML Developer's Guide</title>
+      <genre>Computer</genre>
+      <price>44.95</price>
+      <publish_date>2000-10-01</publish_date>
+      <description>An in-depth look at creating applications 
+      with XML.</description>
+   </book>
+   <book id="bk102">
+      <author>Ralls, Kim</author>
+      <title>Midnight Rain</title>
+      <genre>Fantasy</genre>
+      <price>5.95</price>
+      <publish_date>2000-12-16</publish_date>
+      <description>A former architect battles corporate zombies, 
+      an evil sorceress, and her own childhood to become queen 
+      of the world.</description>
+   </book>
+   <book id="bk103">
+      <author>Corets, Eva</author>
+      <title>Maeve Ascendant</title>
+      <genre>Fantasy</genre>
+      <price>5.95</price>
+      <publish_date>2000-11-17</publish_date>
+      <description>After the collapse of a nanotechnology 
+      society in England, the young survivors lay the 
+      foundation for a new society.</description>
+   </book>
+   <book id="bk104">
+      <author>Corets, Eva</author>
+      <title>Oberon's Legacy</title>
+      <genre>Fantasy</genre>
+      <price>5.95</price>
+      <publish_date>2001-03-10</publish_date>
+      <description>In post-apocalypse England, the mysterious 
+      agent known only as Oberon helps to create a new life 
+      for the inhabitants of London. Sequel to Maeve 
+      Ascendant.</description>
+   </book>
+   <book id="bk105">
+      <author>Corets, Eva</author>
+      <title>The Sundered Grail</title>
+      <genre>Fantasy</genre>
+      <price>5.95</price>
+      <publish_date>2001-09-10</publish_date>
+      <description>The two daughters of Maeve, half-sisters, 
+      battle one another for control of England. Sequel to 
+      Oberon's Legacy.</description>
+   </book>
+   <book id="bk106">
+      <author>Randall, Cynthia</author>
+      <title>Lover Birds</title>
+      <genre>Romance</genre>
+      <price>4.95</price>
+      <publish_date>2000-09-02</publish_date>
+      <description>When Carla meets Paul at an ornithology 
+      conference, tempers fly as feathers get ruffled.</description>
+   </book>
+   <book id="bk107">
+      <author>Thurman, Paula</author>
+      <title>Splish Splash</title>
+      <genre>Romance</genre>
+      <price>4.95</price>
+      <publish_date>2000-11-02</publish_date>
+      <description>A deep sea diver finds true love twenty 
+      thousand leagues beneath the sea.</description>
+   </book>
+   <book id="bk108">
+      <author>Knorr, Stefan</author>
+      <title>Creepy Crawlies</title>
+      <genre>Horror</genre>
+      <price>4.95</price>
+      <publish_date>2000-12-06</publish_date>
+      <description>An anthology of horror stories about roaches,
+      centipedes, scorpions  and other insects.</description>
+   </book>
+</catalog>