Fix current jQuery version

v1.13

.travis.yml

@@ -1,8 +1,9 @@
 language: node_js
 sudo: false
 node_js:
+  - "8.9"
   - "6.2"
-  - "4.0"
+  - "4.7"
 env:
   - CXX=g++-4.8
 addons:

Gruntfile.js

@@ -80,27 +80,11 @@ module.exports = function(grunt) {
             dev: {
                 src: ['front/src/css']
             },
-            coverage: {
-                src: ['.tmp', 'coverage/']
-            },
             build: {
                 src: ['front/build']
             }
         },
         copy: {
-            beforeCoverage: {
-                files: [
-                    {src: ['bin/server.js'], dest: '.tmp/'}
-                ]
-            },
-            coverage: {
-                files: [
-                    {cwd: 'test', src: '**/*', dest: 'coverage/test', expand: true},
-                    {cwd: 'lib/metadata', src: '**/*', dest: 'coverage/lib/metadata', expand: true},
-                    {cwd: 'node_modules/phantomas', src: '**/*', dest: 'coverage/node_modules/phantomas', expand: true},
-                    {cwd: 'lib/tools/phantomas/custom_modules', src: '**/*', dest: 'coverage/lib/tools/phantomas/custom_modules', expand: true}
-                ]
-            },
             build: {
                 files: [
                     {src: ['./front/src/img/favicon.png'], dest: './front/build/img/favicon.png'},
@@ -108,50 +92,18 @@ module.exports = function(grunt) {
                 ]
             }
         },
-        lineremover: {
-            beforeCoverage: {
-                files: {
-                    '.tmp/bin/cli.js': 'bin/cli.js'
-                },
-                options: {
-                    exclusionPattern: /#!\/usr\/bin\/env node/
-                }
-            }
-        },
-        blanket: {
-            coverageApp: {
-                src: ['app/'],
-                dest: 'coverage/app/'
-            },
-            coverageLib: {
-                src: ['lib/'],
-                dest: 'coverage/lib/'
-            },
-            coverageBin: {
-                src: ['.tmp/bin/'],
-                dest: 'coverage/bin/'
-            }
-        },
         mochaTest: {
             test: {
                 options: {
                     reporter: 'spec',
                 },
-                src: ['coverage/test/core/*.js', 'coverage/test/api/*.js']
+                src: ['test/core/*.js', 'test/api/*.js']
             },
             'test-current-work': {
                 options: {
                     reporter: 'spec',
                 },
-                src: ['test/core/imageOptimizerTest.js']
-            },
-            coverage: {
-                options: {
-                    reporter: 'html-cov',
-                    quiet: true,
-                    captureFile: 'coverage/coverage.html'
-                },
-                src: ['coverage/test/core/*.js', 'coverage/test/api/*.js']
+                src: ['test/api/apiTest.js']
             }
         },
         env: {
@@ -182,7 +134,7 @@ module.exports = function(grunt) {
             test: {
                 options: {
                     port: 8387,
-                    server: './coverage/bin/server.js',
+                    server: './bin/server.js',
                     showStack: true
                 }
             },
@@ -243,7 +195,11 @@ module.exports = function(grunt) {
             build: {
                 options: {
                     base: '.tmp',
-                    method: 'append'
+                    method: 'append',
+                    unescape: {
+                        '&lt;': '<',
+                        '&gt;': '>'
+                    }
                 },
                 files: {
                     './front/build/main.html': ['.tmp/views/*.html']
@@ -261,35 +217,11 @@ module.exports = function(grunt) {
         }
     });
 
-
-    // Custom task: copies the test settings.json file to the coverage folder, and checks if there's no missing fields
-    grunt.registerTask('copy-test-server-settings', function() {
-        var mainSettingsFile = './server_config/settings.json';
-        var testSettingsFile = './test/fixtures/settings.json';
-
-        var mainSettings = grunt.file.readJSON(mainSettingsFile);
-        var testSettings = grunt.file.readJSON(testSettingsFile);
-
-        // Recursively compare keys of two objects (not the values)
-        function compareKeys(original, copy, context) {
-            for (var key in original) {
-                if (!copy[key] && copy[key] !== '' && copy[key] !== 0) {
-                    grunt.fail.warn('Settings file ' + testSettingsFile + ' doesn\'t contain key ' + context + '.' + key);
-                }
-                if (original[key] !== null && typeof original[key] === 'object') {
-                    compareKeys(original[key], copy[key], context + '.' + key);
-                }
-            }
-        }
-
-        compareKeys(mainSettings, testSettings, 'settings');
-
-        var outputFile = './coverage/server_config/settings.json';
-        grunt.file.write(outputFile, JSON.stringify(testSettings, null, 4));
-        grunt.log.ok('File ' + outputFile + ' created');
+    // Custom task that sets a variable for tests
+    grunt.registerTask('test-settings', function() {
+        process.env.IS_TEST = true;
     });
 
-
     grunt.registerTask('icons', [
         'webfont:icons',
         'less',
@@ -329,25 +261,18 @@ module.exports = function(grunt) {
     ]);
 
     grunt.registerTask('test', [
+        'test-settings',
         'build',
         'express:testSuite',
-        'clean:coverage',
-        'copy-test-server-settings',
-        'lineremover:beforeCoverage',
-        'copy:beforeCoverage',
-        'blanket',
-        'copy:coverage',
         'express:test',
         'mochaTest:test',
-        'mochaTest:coverage',
         'clean:tmp'
     ]);
 
     grunt.registerTask('test-current-work', [
+        'test-settings',
         'jshint',
         'express:testSuite',
-        'clean:coverage',
-        'copy-test-server-settings',
         'express:test-current-work',
         'mochaTest:test-current-work',
         'clean:tmp'

Vagrantfile

@@ -1,6 +1,6 @@
 Vagrant.configure("2") do |config|
 
-  config.vm.box = "ubuntu/precise64"
+  config.vm.box = "ubuntu/trusty64"
 
   config.vm.network :private_network, ip: "10.10.10.10"
   config.ssh.forward_agent = true
@@ -17,6 +17,8 @@ Vagrant.configure("2") do |config|
     vb.customize ["modifyvm", :id, "--cpus", 2]
   end
 
+  config.vm.synced_folder "./", "/space/YellowLabTools"
+
   config.vm.provision :shell, :path => "server_config/server_install.sh"
 
 end

bin/cli.js

@@ -15,10 +15,14 @@ var cli = meow({
         'Options:',
         '  --device             Use "phone" or "tablet" to simulate a mobile device (by user-agent and viewport size).',
         '  --screenshot         Will take a screenshot and use this value as the output path. It needs to end with ".png".',
-        '  --wait-for-selector  Once the page is loaded, Phantomas will wait until the given CSS selector matches some elements.',
+        //'  --wait-for-selector  Once the page is loaded, Phantomas will wait until the given CSS selector matches some elements.',
+        '  --proxy              Sets an HTTP proxy to pass through. Syntax is "host:port".',
         '  --cookie             Adds a cookie on the main domain.',
         '  --auth-user          Basic HTTP authentication username.',
         '  --auth-pass          Basic HTTP authentication password.',
+        '  --block-domain       Disallow requests to given (comma-separated) domains - aka blacklist.',
+        '  --allow-domain       Only allow requests to given (comma-separated) domains - aka whitelist.',
+        '  --no-externals       Block all domains except the main one.',
         '  --reporter           The output format: "json" or "xml". Default is "json".',
         ''
     ].join('\n'),
@@ -56,6 +60,9 @@ options.device = cli.flags.device || 'desktop';
 // Wait for CSS selector
 options.waitForSelector = cli.flags.waitForSelector || null;
 
+// Proxy
+options.proxy = cli.flags.proxy || null;
+
 // Cookie
 options.cookie = cli.flags.cookie || null;
 
@@ -63,6 +70,11 @@ options.cookie = cli.flags.cookie || null;
 options.authUser = cli.flags.authUser || null;
 options.authPass = cli.flags.authPass || null;
 
+// Domain blocking
+options.blockDomain =  cli.flags.blockDomain || null;
+options.allowDomain =  cli.flags.allowDomain || null;
+options.noExternals =  cli.flags.noExternals || null;
+
 // Output format
 if (cli.flags.reporter && cli.flags.reporter !== 'json' && cli.flags.reporter !== 'xml') {
     console.error('Incorrect parameters: reporter has to be "json" or "xml"');
@@ -80,8 +92,28 @@ if (cli.flags.reporter && cli.flags.reporter !== 'json' && cli.flags.reporter !=
             debug('Success');
             switch(cli.flags.reporter) {
                 case 'xml':
-                    var serializer = new EasyXml();
-                    console.log(serializer.render(data));
+                    var serializer = new EasyXml({
+                        manifest: true
+                    });
+
+                    // Remove some heavy parts of the results object
+                    delete data.toolsResults;
+                    delete data.javascriptExecutionTree;
+
+                    var xmlOutput = serializer.render(data);
+
+                    // Remove special chars from XML tags: # [ ]
+                    xmlOutput = xmlOutput.replace(/<([^>]*)#([^>]*)>/g, '<$1>');
+                    xmlOutput = xmlOutput.replace(/<([^>]*)\[([^>]*)>/g, '<$1>');
+                    xmlOutput = xmlOutput.replace(/<([^>]*)\]([^>]*)>/g, '<$1>');
+
+                    // Remove special chars from text content: \n \0
+                    xmlOutput = xmlOutput.replace(/(<[a-zA-Z]*>[^<]*)\n([^<]*<\/[a-zA-Z]*>)/g, '$1$2');
+                    xmlOutput = xmlOutput.replace(/\0/g, '');
+                    xmlOutput = xmlOutput.replace(/\uFFFF/g, '');
+                    xmlOutput = xmlOutput.replace(/\u0002/g, '');
+
+                    console.log(xmlOutput);
                     break;
                 default:
                     console.log(JSON.stringify(data, null, 2));

front/src/css/rule.css

@@ -231,13 +231,13 @@
   padding-top: 2.5em;
 }
 .totalWeightPie {
-  max-width: 39em;
+  max-width: 20em;
   margin: 2em auto 4em;
 }
 .totalWeightPie canvas {
   max-width: inherit;
 }
-.hugeFile {
+.offenderProblem {
   font-weight: bold;
   color: #e74c3c;
 }

front/src/js/app.js

@@ -35,7 +35,9 @@ yltApp.run(['$rootScope', '$location', function($rootScope, $location) {
 
     // Google Analytics
     $rootScope.$on('$routeChangeSuccess', function(){
-        ga('send', 'pageview', {'page': $location.path()});
+        if (ga) {
+            ga('send', 'pageview', {'page': $location.path()});
+        }
     });
 
     // GitHub star button (asynchronously loaded iframe)

front/src/js/controllers/indexCtrl.js

@@ -1,13 +1,23 @@
 var indexCtrl = angular.module('indexCtrl', []);
 
-indexCtrl.controller('IndexCtrl', ['$scope', 'Settings', 'API', function($scope, Settings, API) {
+indexCtrl.controller('IndexCtrl', ['$scope', '$routeParams', '$location', 'Settings', 'API', function($scope, $routeParams, $location, Settings, API) {
     
     $scope.settings = Settings.getMergedSettings();
 
     $scope.launchTest = function() {
         if ($scope.url) {
+            $location.search('url', null);
+            $location.search('run', null);
             Settings.saveSettings($scope.settings);
             API.launchTest($scope.url, $scope.settings);
         }
     };
+
+    // Auto fill URL field and auto launch test when the good params are set in the URL
+    if ($routeParams.url) {
+        $scope.url = $routeParams.url;
+        if ($routeParams.run === 'true' || $routeParams.run === 1 || $routeParams.run === '1') {
+            $scope.launchTest();
+        }
+    }
 }]);

front/src/js/controllers/ruleCtrl.js

@@ -45,7 +45,23 @@ ruleCtrl.controller('RuleCtrl', ['$scope', '$rootScope', '$routeParams', '$locat
             });
 
             $scope.weightOptions = {
-                tooltipTemplate: '<%=label%>: <%=value%> KB'
+                tooltips: {
+                    callbacks: {
+                        label: function(tooltipItem, data) {
+                            var label = data.labels[tooltipItem.index];
+                            var value = data.datasets[tooltipItem.datasetIndex].data[tooltipItem.index];
+                            return label + ': ' + value + ' KB';
+                        }
+                    }
+                },
+                legend: {
+                    display: true,
+                    position: 'bottom',
+                    labels: {
+                        boxWidth: 12,
+                        fontSize: 14
+                    }
+                }
             };
         }
 
@@ -87,14 +103,27 @@ ruleCtrl.controller('RuleCtrl', ['$scope', '$rootScope', '$routeParams', '$locat
             $scope.breakpointsSeries = ['Number of CSS rules per breakpoint'];
             $scope.breakpointsColours = ['#9c4274'];
             $scope.breakpointsOptions = {
-                scaleShowGridLines: false,
-                barShowStroke: false,
-                showTooltips: false,
-                pointDot: false,
-                responsive: true,
-                maintainAspectRatio: true,
-                strokeColor: 'rgba(20, 200, 20, 1)',
-                scaleFontSize: 9
+                scales: {
+                    xAxes: [{
+                        gridLines: {
+                            display:false
+                        }
+                    }],
+                    yAxes: [{
+                        gridLines: {
+                            display:false
+                        }
+                    }]
+                },
+                tooltips: {
+                    enabled: false
+                },
+
+                elements: {
+                    point: {
+                        radius: 0
+                    }
+                }
             };
         }
     }

front/src/js/directives/offendersDirectives.js

@@ -751,7 +751,7 @@
         }
 
         function onDetailsClick(row) {
-            // Close if it's alreay open
+            // Close if it's already open
             if (row.classList.contains('showDetails')) {
                 closeDetails(row);
                 return;
@@ -918,4 +918,10 @@
         };
     });
 
-})();
+    offendersDirectives.filter('addSpaces', function() {
+        return function(str) {
+            return str.split('').join(' ');
+        };
+    });
+
+})();

front/src/js/services/apiService.js

@@ -11,6 +11,7 @@ apiService.factory('API', ['$location', 'Runs', 'Results', function($location, R
                 screenshot: true,
                 device: settings.device,
                 waitForSelector: settings.waitForSelector,
+                proxy: settings.proxy,
                 cookie: settings.cookie,
                 authUser: settings.authUser,
                 authPass: settings.authPass,
@@ -36,6 +37,8 @@ apiService.factory('API', ['$location', 'Runs', 'Results', function($location, R
             }, function(response) {
                 if (response.status === 429) {
                     alert('Too many requests, you reached the max number of requests allowed in 24h');
+                } else if (response.status === 403) {
+                    alert('This particular query was blocked due to spamming. If you think it\'s an error, please open an issue on GitHub.');
                 } else {
                     alert('An error occured...');
                 }

front/src/less/rule.less

@@ -253,7 +253,7 @@
 }
 
 .totalWeightPie {
-    max-width: 39em;
+    max-width: 20em;
     margin: 2em auto 4em;
 
     canvas {
@@ -261,7 +261,7 @@
     }
 }
 
-.hugeFile {
+.offenderProblem {
     font-weight: bold;
     color: #e74c3c;
 }

front/src/main.html

@@ -16,7 +16,6 @@
     <link rel="stylesheet" type="text/css" href="css/screenshot.css">
     <link rel="stylesheet" type="text/css" href="css/timeline.css">
     <link rel="stylesheet" type="text/css" href="css/about.css">
-    <link rel="stylesheet" type="text/css" href="node_modules/angular-chart.js/dist/angular-chart.css">
     <!-- endbuild -->
 
     <link rel="preconnect" href="//www.google-analytics.com">
@@ -26,7 +25,7 @@
     <link rel="preconnect" href="https://api.github.com">
     <link rel="dns-prefetch" href="https://api.github.com">
 
-<head>
+</head>
 
 <body ng-app="YellowLabTools">
     <div id="header"><h1>Yellow Lab <span class="icon-lab"></span> Tools</h1></div>
@@ -39,7 +38,7 @@
 
     <!-- build:js js/all.js -->
     <script src="node_modules/angular/angular.min.js"></script>
-    <script src="node_modules/chart.js/Chart.min.js"></script>
+    <script src="node_modules/chart.js/dist/Chart.min.js"></script>
     <script src="node_modules/angular-route/angular-route.min.js"></script>
     <script src="node_modules/angular-resource/angular-resource.min.js"></script>
     <script src="node_modules/angular-sanitize/angular-sanitize.min.js"></script>
@@ -64,7 +63,7 @@
     <!-- endbuild -->
 
     <script>
-        (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)})(window,document,'script','//www.google-analytics.com/analytics.js','ga');if('@@googleAnalyticsId'.indexOf('UA-')===0){ga('create','@@googleAnalyticsId','auto');}
+        if('@@googleAnalyticsId'.indexOf('UA-')===0){(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)})(window,document,'script','https://www.google-analytics.com/analytics.js','ga');ga('create','@@googleAnalyticsId','auto');}
     </script>
 </body>
 </html>

front/src/views/dashboard.html

@@ -3,7 +3,7 @@
     
     <div ng-if="result.blockedRequests">
         <b><ng-pluralize count="result.blockedRequests.length" when="{'0': 'No blocked request', 'one': '1 blocked request', 'other': '{} blocked requests'}"></ng-pluralize>:</b>
-        <div ng-repeat="request in result.blockedRequests">
+        <div ng-repeat="request in result.blockedRequests track by $index">
             {{request}}
         </div>
     </div>
@@ -17,7 +17,7 @@
             </div>
         </div>
         <div>
-            <a href="/result/{{result.runId}}/screenshot">
+            <a href="result/{{result.runId}}/screenshot">
                 <div class="screenshotWrapper" ng-class="result.params.options.device || 'desktop'">
                     <div>
                         <img ng-if="result.screenshotUrl" class="screenshotImage" ng-src="{{result.screenshotUrl}}"/>
@@ -36,7 +36,7 @@
             <div class="criteria">
                 <div class="table" title="Click to see details">
                     <a ng-repeat="ruleName in category.rules" ng-if="result.rules[ruleName]" ng-init="rule = result.rules[ruleName]"
-                         ng-class="{'warning': rule.abnormal}" href="/result/{{runId}}/rule/{{ruleName}}">
+                         ng-class="{'warning': rule.abnormal}" href="result/{{runId}}/rule/{{ruleName}}">
                         <div class="grade">
                             <grade score="rule.score"></grade>
                         </div>

front/src/views/index.html

@@ -15,11 +15,13 @@
             <span ng-if="!settings.showAdvanced">Advanced settings &nbsp;✚</span>
             <span ng-if="settings.showAdvanced">Hide advanced settings &nbsp;✖</span>
         </a> ]
-        <span class="currentSettings" ng-if="!settings.showAdvanced && (settings.waitForSelector || settings.cookie || settings.authUser || settings.authPass)">
+        <span class="currentSettings" ng-if="!settings.showAdvanced && (settings.waitForSelector || settings.cookie || settings.authUser || settings.authPass || settings.proxy || settings.domains)">
             Currently set:
             <span ng-if="settings.waitForSelector">wait for selector</span>
             <span ng-if="settings.cookie">cookie</span>
             <span ng-if="settings.authUser || settings.authPass">authentication</span>
+            <span ng-if="settings.proxy">proxy</span>
+            <span ng-if="settings.domains">domain blocking</span>
         </span>
         <div class="advanced" ng-show="settings.showAdvanced">
             <!--<div>
@@ -37,7 +39,7 @@
                     Cookie
                     <span class="settingsTooltip">
                         <span class="icon-question"></span>
-                        <div><b>Cookie</b><br><br>Adds a cookie on the main domain.<br><br>Example: "bar=foo;domain=url"</div>
+                        <div><b>Cookie</b><br><br>Adds cookies, separated by a pipe character.<br><br>Example: "bar1=foo1;domain=.domain1.com|bar2=foo2;domain=www.domain2.com"</div>
                     </span>
                 </div>
                 <div><input type="text" name="cookie" ng-model="settings.cookie" /></div>
@@ -61,6 +63,16 @@
                     </div>
                 </div>
             </div>
+            <div>
+                <div class="label">
+                    HTTP proxy
+                    <span class="settingsTooltip">
+                        <span class="icon-question"></span>
+                        <div><b>HTTP proxy</b><br><br>Insert here your proxy settings with the format "host:port".<br><br>Example: "192.168.10.0:3333"</div>
+                    </span>
+                </div>
+                <div><input type="text" name="proxy" ng-model="settings.proxy" /></div>
+            </div>
             <div>
                 <div class="label">
                     Block domains

front/src/views/rule.html

@@ -117,10 +117,6 @@
                         <file-and-line-button file="offender.file" line="offender.line" column="offender.column"></file-and-line-button>
                     </div>
 
-                    <div ng-if="policyName === 'cssDuplicatedSelectors'">
-                        {{offender.rule}} (<b>x{{offender.occurrences}}</b>)
-                    </div>
-
                     <div ng-if="policyName === 'cssOldPropertyPrefixes'">
                         <b>{{offender.property}} {{offender.message}}</b>
                         <div ng-if="offender.rules.length" ng-click="offender.showMore = !offender.showMore" class="offenderButton">
@@ -158,16 +154,21 @@
                     <div ng-if="policyName === 'globalVariables' || policyName === 'jQueryVersionsLoaded' || policyName === 'synchronousXHR'">
                         {{offender}}
                     </div>
+
+                    <div ng-if="policyName === 'fontsCount'">
+                        <url-link url="offender.url" max-length="70"></url-link>
+                        ({{offender.size | bytes}})
+                    </div>
                 </div>
             </div>
         </div>
 
-        <div ng-repeat="(file, fileDetails) in rule.offendersObj.byFile track by $index">
+        <div ng-repeat="fileDetails in rule.offendersObj.byFile track by $index">
             <h3>
                 <ng-pluralize count="fileDetails.count" when="{'one': '1 offender', 'other': '{} offenders'}"></ng-pluralize>
                 in
-                <url-link ng-if="file !== 'Inline CSS'" url="file" max-length="80"></url-link>
-                <span ng-if="file === 'Inline CSS'">inline CSS</span>
+                <url-link ng-if="fileDetails.url !== 'Inline CSS'" url="fileDetails.url" max-length="80"></url-link>
+                <span ng-if="fileDetails.url === 'Inline CSS'">inline CSS</span>
             </h3>
 
             <div class="offendersTable">
@@ -183,6 +184,10 @@
                         <span ng-if="offender.line !== null && offender.column !== null"> @ {{offender.line}}:{{offender.column}}</span>
                     </div>
 
+                    <div ng-if="policyName === 'cssDuplicatedSelectors'">
+                        {{offender.rule}} (<b>x{{offender.occurrences}}</b>)
+                    </div>
+
                     <div ng-if="policyName === 'cssDuplicatedProperties'">
                         Property <b>{{offender.property}}</b> duplicated in <b>{{offender.rule}} { }</b>
                         <span ng-if="offender.line !== null && offender.column !== null"> @ {{offender.line}}:{{offender.column}}</span>
@@ -243,14 +248,14 @@
     <div ng-if="policyName === 'totalWeight'">
         <h3>Weight by MIME type</h3>
         <div class="totalWeightPie">
-            <canvas class="chart chart-doughnut" data="weightData" labels="weightLabels" options="weightOptions" colours="weightColours" legend="true"></canvas>
+            <canvas class="chart chart-doughnut" chart-data="weightData" chart-labels="weightLabels" chart-options="weightOptions" chart-colors="weightColours"></canvas>
         </div>
         <div ng-repeat="type in weightLabels">
             <h3>{{rule.offendersObj.list.byType[type].totalWeight | bytes}} of {{type}}</h3>
             <div class="offendersTable">
                 <div ng-repeat="request in rule.offendersObj.list.byType[type].requests | orderBy:'-weight'" ng-if="request.weight > 0">
                     <div><url-link url="request.url" max-length="60"></url-link></div>
-                    <div ng-class="{hugeFile: request.weight > 102400}">{{request.weight | bytes}}</div>
+                    <div ng-class="{offenderProblem: request.weight > 102400}">{{request.weight | bytes}}</div>
                 </div>
             </div>
         </div>
@@ -369,28 +374,28 @@
 
     <div ng-if="policyName === 'DOMaccesses'">
         <h3>{{rule.value}} offenders</h3>
-        Please open the <a href="/result/{{runId}}/timeline">JS timeline</a>
+        Please open the <a href="result/{{runId}}/timeline">JS timeline</a>
     </div>
 
     <div ng-if="policyName === 'queriesWithoutResults'">
         <h3>{{rule.value}} offenders</h3>
-        Please open the <a href="/result/{{runId}}/timeline#filter=queryWithoutResults">JS timeline, filtered by "Queries without results"</a>
+        Please open the <a href="result/{{runId}}/timeline#filter=queryWithoutResults">JS timeline, filtered by "Queries without results"</a>
     </div>
 
     <div ng-if="policyName === 'jQueryCallsOnEmptyObject'">
         <h3>{{rule.value}} offenders</h3>
-        Please open the <a href="/result/{{runId}}/timeline#filter=jQueryCallOnEmptyObject">JS timeline, filtered by "jQuery calls on empty object"</a>
+        Please open the <a href="result/{{runId}}/timeline#filter=jQueryCallOnEmptyObject">JS timeline, filtered by "jQuery calls on empty object"</a>
     </div>
 
     <div ng-if="policyName === 'jQueryNotDelegatedEvents'">
         <h3>{{rule.value}} offenders</h3>
-        Please open the <a href="/result/{{runId}}/timeline#filter=eventNotDelegated">JS timeline, filtered by "Events not delegated"</a>
+        Please open the <a href="result/{{runId}}/timeline#filter=eventNotDelegated">JS timeline, filtered by "Events not delegated"</a>
     </div>
 
     <div ng-if="policyName === 'cssBreakpoints'">
         <div ng-if="rule.value > 0" class="cssBreakpointsGraph">
             <h3>Breakpoints distribution graph</h3>
-            <canvas class="chart chart-line" chart-data="breakpointsData" chart-labels="breakpointsLabels" chart-options="breakpointsOptions" chart-legend="true" chart-colours="breakpointsColours" chart-series="breakpointsSeries" width="600" height="250"></canvas>
+            <canvas class="chart chart-line" chart-data="breakpointsData" chart-labels="breakpointsLabels" chart-options="breakpointsOptions" chart-colors="breakpointsColours" chart-series="breakpointsSeries" width="600" height="250"></canvas>
             <h3>Breakpoints list</h3>
             <div class="offendersTable">
                 <div ng-repeat="offender in rule.offendersObj | orderBy:'pixels'">
@@ -403,6 +408,62 @@
         </div>
     </div>
 
+    <div ng-if="policyName === 'heavyFonts'">
+        <div ng-repeat="font in rule.offendersObj.fonts | orderBy:'-weight' track by $index">
+            <h3><url-link url="font.url" max-length="80"></url-link></h3>
+            <div class="offendersTable">
+                <div>
+                    <div>Weight</div>
+                    <div ng-if="font.weight <= 40960">{{font.weight | bytes}}</div>
+                    <div ng-if="font.weight > 40960" class="offenderProblem">{{font.weight | bytes}}</div>
+                </div>
+                <div>
+                    <div>Number of glyphs</div>
+                    <div ng-if="font.numGlyphs <= 500">{{font.numGlyphs}}</div>
+                    <div ng-if="font.numGlyphs > 500" class="offenderProblem">{{font.numGlyphs}} (better &lt; 500)</div>
+                </div>
+                <div>
+                    <div>Average glyph complexity</div>
+                    <div ng-if="font.averageGlyphComplexity <= 35">{{font.averageGlyphComplexity}}</div>
+                    <div ng-if="font.averageGlyphComplexity > 35" class="offenderProblem">{{font.averageGlyphComplexity}} (better &lt; 35)</div>
+                </div>
+            </div>
+        </div>
+    </div>
+
+    <div ng-if="policyName === 'unusedUnicodeRanges'">
+        <div ng-repeat="font in rule.offendersObj.fonts | orderBy:'-compressedWeigth' track by $index">
+            <h3><url-link url="font.url" max-length="60"></url-link> ({{font.weight | bytes}})</h3>
+            <div ng-if="font.isIconFont" class="offendersTable">
+                <div>
+                    <div>
+                        This font seems to be an icon font
+                        <span ng-if="font.numGlyphsInCommonWithPageContent / font.glyphs <= 0.05" class="offenderProblem">but only {{font.numGlyphsInCommonWithPageContent}} of its {{font.glyphs}} glyphs <ng-pluralize count="font.numGlyphsInCommonWithPageContent" when="{'one': 'is', 'other': 'are'}"></ng-pluralize> possibly used!</span>
+                        <span ng-if="font.numGlyphsInCommonWithPageContent / font.glyphs > 0.05">and {{font.numGlyphsInCommonWithPageContent}} of its {{font.glyphs}} glyphs <ng-pluralize count="font.numGlyphsInCommonWithPageContent" when="{'one': 'is', 'other': 'are'}"></ng-pluralize> possibly used.</span>
+                    </div>
+                </div>
+            </div>
+            <div ng-if="!font.isIconFont" class="offendersTable">
+                <div ng-repeat="range in font.unicodeRanges track by $index">
+                    <div><b>{{range.name}}</b></div>
+                    <div ng-if="!range.underused">{{range.numGlyphsInCommonWithPageContent}} of its {{range.charset.length}} glyphs <ng-pluralize count="range.numGlyphsInCommonWithPageContent" when="{'one': 'is', 'other': 'are'}"></ng-pluralize> possibly used</div>
+                    <div ng-if="range.underused" class="offenderProblem">{{range.numGlyphsInCommonWithPageContent}} of its {{range.charset.length}} glyphs are used</div>
+                    <div>
+                        <div class="offenderButton opens">
+                            glyphes list
+                            <div>{{range.charset | addSpaces}}</div>
+                        </div>
+                    </div>
+                </div>
+                <div ng-if="font.ligaturesOrHiddenChars > 0">
+                    <div><b>Ligatures or hidden chars</b></div>
+                    <div ng-class="{offenderProblem: (font.ligaturesOrHiddenChars > 25)}">{{font.ligaturesOrHiddenChars}} glyphs</div>
+                    <div></div>
+                </div>
+            </div>
+        </div>
+    </div>
+
     <div ng-if="policyName === 'http2'">
         <h3>Protocols advertised by the server</h3>
         <div class="offendersTable">

lib/metadata/policies.js

@@ -6,18 +6,18 @@ var policies = {
         "tool": "phantomas",
         "label": "DOM elements count",
         "message": "<p>A high number of DOM elements means a lot of work for the browser to render the page.</p><p>It also slows down JavaScript DOM queries, as there are more elements to search through.</p>",
-        "isOkThreshold": 1000,
-        "isBadThreshold": 2500,
-        "isAbnormalThreshold": 4000,
+        "isOkThreshold": 1500,
+        "isBadThreshold": 3000,
+        "isAbnormalThreshold": 4500,
         "hasOffenders": false
     },
     "DOMelementMaxDepth": {
         "tool": "phantomas",
         "label": "DOM max depth",
         "message": "<p>A deep DOM makes the CSS matching with DOM elements difficult.</p><p>It also slows down JavaScript modifications to the DOM because changing the dimensions of an element makes the browser re-calculate the dimensions of it's parents. Same thing for JavaScript events, that bubble up to the document root.</p>",
-        "isOkThreshold": 10,
-        "isBadThreshold": 20,
-        "isAbnormalThreshold": 28,
+        "isOkThreshold": 12,
+        "isBadThreshold": 22,
+        "isAbnormalThreshold": 30,
         "hasOffenders": true,
         "offendersTransformFn": function(offenders) {
             var domArrays = offenders.map(offendersHelpers.domPathToArray);
@@ -32,7 +32,7 @@ var policies = {
         "tool": "phantomas",
         "label": "Number of iframes",
         "message": "<p>iFrames are the most complex HTML elements. They are pages, just like the main page, and the browser needs to create a new page context, which has a cost.</p>",
-        "isOkThreshold": 2,
+        "isOkThreshold": 3,
         "isBadThreshold": 15,
         "isAbnormalThreshold": 30,
         "hasOffenders": false
@@ -70,9 +70,9 @@ var policies = {
         "tool": "jsExecutionTransformer",
         "label": "DOM access",
         "message": "<p>This metric counts the number of calls to DOM related functions (both native DOM functions and jQuery functions) on page load.</p><p>The more your JavaScript code accesses the DOM, the slower the page will load.</p><p>Try, as much as possible, to have an HTML page fully generated by the server instead of making changes with JS.</p><p>Try to reduce the number of queries by refactoring your JavaScript code.</p><p>Binding too many events also has a cost. Try to use <a href=\"https://learn.jquery.com/events/event-delegation/\" target=\"_blank\">event delegation</a> as much as possible.</p>",
-        "isOkThreshold": 50,
-        "isBadThreshold": 1500,
-        "isAbnormalThreshold": 3000,
+        "isOkThreshold": 200,
+        "isBadThreshold": 2000,
+        "isAbnormalThreshold": 4000,
         "hasOffenders": false
     },
     "queriesWithoutResults": {
@@ -80,8 +80,8 @@ var policies = {
         "label": "Queries without result",
         "message": "<p>Number of queries that return no result. Both native and jQuery DOM requests are counted.</p><p>It suggests the query is not used on the page, probably because it is some dead code.</p><p>Or maybe the code is trying to find an HTML block that is not always here. Look at the JS Timeline to see if the scripts correctly figures out the HTML block is not here and immediatly stops interacting further with the DOM.</p>",
         "isOkThreshold": 0,
-        "isBadThreshold": 100,
-        "isAbnormalThreshold": 200,
+        "isBadThreshold": 150,
+        "isAbnormalThreshold": 250,
         "hasOffenders": false
     },
     "DOMqueriesAvoidable": {
@@ -89,8 +89,8 @@ var policies = {
         "label": "Duplicated DOM queries",
         "message": "<p>This is the number of queries that could be avoided by removing all duplicated queries.</p><p>Simply save the result of a query in a variable. Ok it is not always simple, especially with third-party scripts, but at least do it with your own code.</p>",
         "isOkThreshold": 0,
-        "isBadThreshold": 200,
-        "isAbnormalThreshold": 500,
+        "isBadThreshold": 300,
+        "isAbnormalThreshold": 600,
         "hasOffenders": true,
         "takeOffendersFrom": "DOMqueriesDuplicated",
         "offendersTransformFn": function(offenders) {
@@ -121,7 +121,7 @@ var policies = {
         "label": "Scroll events bound",
         "message": "<p>Number of 'scroll' event listeners binded to 'window' or 'document'.</p><p>Asking too much work to the browser on scroll hurts the smoothness of the scroll. Merging all your event listeners into an unique listener can help you factorize their code and reduce their footprint on scroll.</p>",
         "isOkThreshold": 1,
-        "isBadThreshold": 7,
+        "isBadThreshold": 8,
         "isAbnormalThreshold": 15,
         "hasOffenders": true,
         "offendersTransformFn": function(offenders) {
@@ -195,8 +195,8 @@ var policies = {
         "label": "document.write calls",
         "message": "<p>They slow down the page construction, especially if they are used to insert scripts in the page. Remove them ASAP.</p><p>If you cannot remove them because they come from a third-party script (such as ads), have a look at <a href=\"https://github.com/krux/postscribe\" target=\"_blank\">PostScribe</a>.</p>",
         "isOkThreshold": 0,
-        "isBadThreshold": 5,
-        "isAbnormalThreshold": 10,
+        "isBadThreshold": 3,
+        "isAbnormalThreshold": 8,
         "hasOffenders": true,
         "offendersTransformFn": function(offenders) {
             return {
@@ -254,8 +254,8 @@ var policies = {
         "tool": "phantomas",
         "label": "Console messages",
         "message": "<p>Try to keep your console clean when in production. Debugging is good for development only.</p><p>Writing in the console has a cost, especially when dumping large object variables.</p><p>There is also a problem with Internet Explorer 8, not knowing the console object.</p>",
-        "isOkThreshold": 0,
-        "isBadThreshold": 10,
+        "isOkThreshold": 3,
+        "isBadThreshold": 20,
         "isAbnormalThreshold": 50,
         "hasOffenders": false
     },
@@ -276,7 +276,7 @@ var policies = {
     },
     "jQueryVersion": {
         "label": "jQuery version",
-        "message": "<p>The current latest version of jQuery is 3.0</p><p>Each new version of jQuery optimizes performances. Do not keep an old version of jQuery. Updating can sometimes break a few things, but it is generally quite easy to fix them up. So don't hesitate.</p>",
+        "message": "<p>The current latest version of jQuery is 3.3</p><p>Each new version of jQuery optimizes performances. Do not keep an old version of jQuery. Updating can sometimes break a few things, but it is generally quite easy to fix them up. So don't hesitate.</p>",
         "hasOffenders": false,
         "scoreFn": function(data) {
             var differentVersions = data.toolsResults.phantomas.metrics.jQueryVersionsLoaded;
@@ -288,28 +288,33 @@ var policies = {
                 var value = data.toolsResults.phantomas.metrics.jQueryVersion;
                 var score;
 
-                if (value.indexOf('3.0.') === 0 ||
-                    value.indexOf('3.1.') === 0 ||
-                    value.indexOf('3.2.') === 0) {
+                if (value.indexOf('3.3.') === 0 ||
+                    value.indexOf('3.4.') === 0 ||
+                    value.indexOf('3.5.') === 0 ||
+                    value.indexOf('4.0.') === 0) {
                     score = 100;
+                } else if (value.indexOf('3.2.') === 0) {
+                    score = 90;
+                } else if (value.indexOf('3.1.') === 0) {
+                    score = 70;
+                } else if (value.indexOf('3.0.') === 0) {
+                    score = 50;
                 } else if (value.indexOf('1.12.') === 0 ||
                            value.indexOf('2.2.') === 0) {
-                    score = 90;
+                    score = 40;
                 } else if (value.indexOf('1.11.') === 0 ||
                            value.indexOf('2.1.') === 0) {
-                    score = 70;
+                    score = 30;
                 } else if (value.indexOf('1.10.') === 0 ||
                            value.indexOf('2.0.') === 0) {
-                    score = 50;
-                } else if (value.indexOf('1.9.') === 0) {
-                    score = 40;
-                } else if (value.indexOf('1.8.') === 0) {
-                    score = 30;
-                } else if (value.indexOf('1.7') === 0) {
                     score = 20;
-                } else if (value.indexOf('1.6') === 0) {
+                } else if (value.indexOf('1.9') === 0) {
                     score = 10;
-                } else if (value.indexOf('1.5') === 0 ||
+                } else if (value.indexOf('1.9') === 0 ||
+                           value.indexOf('1.8') === 0 ||
+                           value.indexOf('1.7') === 0 ||
+                           value.indexOf('1.6') === 0 ||
+                           value.indexOf('1.5') === 0 ||
                            value.indexOf('1.4') === 0 ||
                            value.indexOf('1.3') === 0 ||
                            value.indexOf('1.2') === 0) {
@@ -343,15 +348,6 @@ var policies = {
         "isAbnormalThreshold": 2,
         "hasOffenders": true
     },
-    "jQueryFunctionsUsed": {
-        "tool": "jsExecutionTransformer",
-        "label": "jQuery usage",
-        "message": "<p>This is the number of different core jQuery functions called on load. This rule is not trying to blame you for using jQuery too much, but the opposite.</p><p>If only a few functions are used, why not trying to get rid of jQuery? Have a look at <a href=\"http://youmightnotneedjquery.com/\" target=\"_blank\">http://youmightnotneedjquery.com</a>.</p>",
-        "isOkThreshold": 15,
-        "isBadThreshold": 6,
-        "isAbnormalThreshold": 0,
-        "hasOffenders": true
-    },
     "jQueryCallsOnEmptyObject": {
         "tool": "jsExecutionTransformer",
         "label": "Calls on empty objects",
@@ -427,7 +423,7 @@ var policies = {
         "tool": "phantomas",
         "label": "Rules count",
         "message": "<p>Having a huge number of CSS rules hurts performances. If the number of CSS rules is higher than the number of DOM elements, there is clearly a problem.</p><p>Huge stylesheets generally occur when the different pages of a website load all the CSS, concatenated in a single stylesheet, even if a large part of the rules are page-specific. Solution is to create one main CSS file with global rules and one custom file per page.</p>",
-        "isOkThreshold": 750,
+        "isOkThreshold": 1000,
         "isBadThreshold": 3000,
         "isAbnormalThreshold": 4500,
         "hasOffenders": true,
@@ -470,7 +466,7 @@ var policies = {
         "label": "Complex selectors",
         "message": "<p>Complex selectors are CSS selectors with 4 or more expressions, like \"#header ul li .foo\".</p><p>They are adding more work for the browser, and this could be avoided by simplifying selectors. The <a href=\"http://getbem.com\" target=\"_blank\">B.E.M. methodology</a> is an useful way to simplify your CSS.</p>",
         "isOkThreshold": 0,
-        "isBadThreshold": 600,
+        "isBadThreshold": 800,
         "isAbnormalThreshold": 2000,
         "hasOffenders": true,
         "offendersTransformFn": function(offenders) {
@@ -486,7 +482,7 @@ var policies = {
         "tool": "phantomas",
         "label": "Colors count",
         "message": "<p>This is the number of different colors defined in CSS.</p><p>Your CSS will be easier to maintain if you keep a small color set.</p>",
-        "isOkThreshold": 30,
+        "isOkThreshold": 50,
         "isBadThreshold": 150,
         "isAbnormalThreshold": 400,
         "hasOffenders": true,
@@ -569,8 +565,8 @@ var policies = {
         "tool": "mediaQueriesChecker",
         "label": "Not mobile-first media queries",
         "message": "<p>This is the number of CSS rules inside media queries that address small screens.</p><p>The common good practice, when creating a responsive website, is to write it \"mobile-first\". More explanation in <a href=\"http://www.sitepoint.com/introduction-mobile-first-media-queries\" target=\"_blank\">this great article</a>.</p>",
-        "isOkThreshold": 25,
-        "isBadThreshold": 200,
+        "isOkThreshold": 50,
+        "isBadThreshold": 250,
         "isAbnormalThreshold": 1000,
         "hasOffenders": true,
         "offendersTransformFn": function(offenders) {
@@ -604,24 +600,24 @@ var policies = {
         "isAbnormalThreshold": 100,
         "hasOffenders": true,
         "offendersTransformFn": function(offenders) {
-            return {
-                count: offenders.length,
-                list: offenders.map(function(offender) {
-                    var parts = /^(.*) \((\d+) times\)$/.exec(offender);
-
-                    if (!parts) {
-                        debug('cssDuplicatedSelectors offenders transform function error with "%s"', offender);
-                        return {
-                            parseError: offender
-                        };
-                    }
+            var parsedOffenders = offenders.map(function(offender) {
+                var parts = /^(.*) \((\d+) times\) ?<(.*)>$/.exec(offender);
 
+                if (!parts) {
+                    debug('cssDuplicatedSelectors offenders transform function error with "%s"', offender);
                     return {
-                        rule: parts[1],
-                        occurrences: parseInt(parts[2], 10)
+                        parseError: offender
                     };
-                })
-            };
+                }
+
+                return {
+                    rule: parts[1],
+                    occurrences: parseInt(parts[2], 10),
+                    file: parts[3]
+                };
+            });
+
+            return offendersHelpers.orderByFile(parsedOffenders);
         }
     },
     "cssDuplicatedProperties": {
@@ -743,8 +739,8 @@ var policies = {
         "label": "Old IE fixes",
         "message": "<p>What browser do you need to support? Once you've got the answer, take a look at these old rules that pollute your CSS code and remove them.</p><p>IE6:<ul><li>* html</li><li>html > body (everything but IE6)</li></ul><p><p>IE7:<ul><li><b>*</b>height: 123px;</li><li>height: 123px <b>!ie</b>;</li></ul><p><p>IE9:<ul><li>-ms-filter</li><li>progid:DXImageTransform.Microsoft</li></ul></p>",
         "isOkThreshold": 0,
-        "isBadThreshold": 75,
-        "isAbnormalThreshold": 300,
+        "isBadThreshold": 50,
+        "isAbnormalThreshold": 150,
         "hasOffenders": true,
         "offendersTransformFn": function(offenders) {
             var parsedOffenders = offenders.map(function(offender) {
@@ -799,8 +795,8 @@ var policies = {
         "label": "Old prefixes",
         "message": "<p>Many property prefixes such as -moz- or -webkit- are not needed anymore, or by very few people. Sometimes, they have never even existed. You can remove them or replace them with the non-prefixed version. This will help reducing your stylesheets weight.</p><p>The prefixes database comes from <a href=\"http://caniuse.com/\" target=\"_blank\">Can I Use</a>.</p>",
         "isOkThreshold": 0,
-        "isBadThreshold": 75,
-        "isAbnormalThreshold": 300,
+        "isBadThreshold": 50,
+        "isAbnormalThreshold": 200,
         "hasOffenders": true,
         "offendersTransformFn": function(offenders) {
             var properties = {};
@@ -901,10 +897,10 @@ var policies = {
     "totalWeight": {
         "tool": "redownload",
         "label": "Total weight",
-        "message": "<p>The weight is of course very important if you want the page to load fast. Try to stay under 1MB, which is alreay very long to download over a slow connection.</p><p>Please note that Yellow Lab Tools' engine (PhantomJS) is not compatible with image srcset (unless you use a polyfill). This can lead to incorrect page weight.</p>",
-        "isOkThreshold": 716800,
-        "isBadThreshold": 2097152,
-        "isAbnormalThreshold": 3145728,
+        "message": "<p>The weight is of course very important if you want the page to load fast. Try to stay under 1MB, which is already very long to download over a slow connection.</p><p>Please note that Yellow Lab Tools' engine (PhantomJS) is not compatible with image srcset (unless you use a polyfill). This can lead to incorrect page weight.</p>",
+        "isOkThreshold": 1048576,
+        "isBadThreshold": 2621440,
+        "isAbnormalThreshold": 4194304,
         "hasOffenders": true,
         "unit": 'bytes'
     },
@@ -912,8 +908,8 @@ var policies = {
         "tool": "redownload",
         "label": "Image optimization",
         "message": "<p>This metric measures the number of bytes that could be saved by optimizing images.</p><p>Image optimization is generally one of the easiest way to reduce a page weight, and as a result, the page load time. Don't use Photoshop or other image editing tools, they're not very good for optimization. Use specialized tools such as <a href=\"https://kraken.io/\" target=\"_blank\">Kraken.io</a> or the excellent <a href=\"https://imageoptim.com/\" target=\"_blank\">ImageOptim</a> on Mac. For SVG images, you can use <a href=\"https://jakearchibald.github.io/svgomg/\" target=\"_blank\">SVGOMG</a></p><p>The tools in use in YellowLabTools are not set to their maximum optimization power (JPEG quality 85), so you might be able to compress even more!</p><p>Please note that Yellow Lab Tools' engine (PhantomJS) is not compatible with image srcset (unless you use a polyfill). This can lead to incorrect page weight.</p>",
-        "isOkThreshold": 10240,
-        "isBadThreshold": 122880,
+        "isOkThreshold": 20480,
+        "isBadThreshold": 204800,
         "isAbnormalThreshold": 307200,
         "hasOffenders": true,
         "unit": 'bytes'
@@ -941,10 +937,10 @@ var policies = {
     "totalRequests": {
         "tool": "redownload",
         "label": "Requests number",
-        "message": "<p>This is one of the most important performance rule. Every request is slowing down the page loading.</p><p>There are several technics to reduce their number:<ul><li>Concatenate JS files</li><li>Concatenate CSS files</li><li>Embed or inline small JS or CSS files in the HTML</li><li>Create sprites or icon fonts</li><li>Base64 encode small images in HTML or stylesheets</li><li>Use lazyloading for images</li></ul></p>",
-        "isOkThreshold": 15,
-        "isBadThreshold": 100,
-        "isAbnormalThreshold": 180,
+        "message": "<p>This is one of the most important performance rule. Every request is slowing down the page loading.</p><p>There are several technics to reduce their number:<ul><li>Concatenate JS files</li><li>Concatenate CSS files</li><li>Embed or inline small JS or CSS files in the HTML</li><li>Create sprites</li><li>Base64 encode small images in HTML or stylesheets</li><li>Use lazyloading for images</li></ul></p>",
+        "isOkThreshold": 20,
+        "isBadThreshold": 120,
+        "isAbnormalThreshold": 200,
         "hasOffenders": true
     },
     "domains": {
@@ -990,8 +986,8 @@ var policies = {
         "label": "Connections closed",
         "message": "<p>This counts the number of requests not keeping the connection alive (specifying \"Connection: close\" in the response headers). It is only counting a request if it is followed by another request on the same domain.</p><p>This is slowing down the next request, because the brower needs to open a new connection to the server, which means an additional round-trip.</p><p>Correct the problem by setting a Keep-Alive header on the guilty server.</p>",
         "isOkThreshold": 0,
-        "isBadThreshold": 7,
-        "isAbnormalThreshold": 20,
+        "isBadThreshold": 4,
+        "isAbnormalThreshold": 15,
         "hasOffenders": true
     },
     "identicalFiles": {
@@ -1019,9 +1015,9 @@ var policies = {
         "tool": "redownload",
         "label": "Small requests",
         "message": "<p>List of all requests that are less than 2 KB. Try to merge them with other files.</p>",
-        "isOkThreshold": 4,
-        "isBadThreshold": 30,
-        "isAbnormalThreshold": 50,
+        "isOkThreshold": 10,
+        "isBadThreshold": 50,
+        "isAbnormalThreshold": 80,
         "hasOffenders": true
     },
     "lazyLoadableImagesBelowTheFold": {
@@ -1042,6 +1038,43 @@ var policies = {
         "isAbnormalThreshold": 30,
         "hasOffenders": true
     },
+    "fontsCount": {
+        "tool": "redownload",
+        "label": "Webfonts number",
+        "message": "<p>This is the number of custom web fonts loaded on the page.</p><p>Webfonts are beautiful, but heavy. You should keep their number as low as possible.</p>",
+        "isOkThreshold": 1,
+        "isBadThreshold": 5,
+        "isAbnormalThreshold": 7,
+        "hasOffenders": true,
+        "offendersTransformFn": function(offenders) {
+            return offenders;
+        }
+    },
+    "heavyFonts": {
+        "tool": "redownload",
+        "label": "Overweighted webfonts",
+        "message": "<p>This metric is the sum of all bytes above 40KB in loaded fonts. Over this size, the font is probably not optimized for the web.</p><p>It can be a compresson issue, a font that contains too many glyphs or a font with complex shapes.</p><p>Sorry, Yellow Lab Tools is not yet compatible with the WOFF2 font format that generates 20-30% smaller fonts. You can proceed to a manual verification on a modern browser.</p>",
+        "isOkThreshold": 0,
+        "isBadThreshold": 102400,
+        "isAbnormalThreshold": 204800,
+        "unit": 'bytes',
+        "hasOffenders": true,
+        "offendersTransformFn": function(offenders) {
+            return offenders;
+        }
+    },
+    "unusedUnicodeRanges": {
+        "tool": "redownload",
+        "label": "Unused Unicode ranges",
+        "message": "<p>This metric counts the number of unused Unicode ranges inside each font. For example, one font could include Cyrillic glyphs but none of them are used on the page.</p><p>It also reveals the number of ligatures (letters that are represented differently when close to each other) and hidden chars (glyphs not linked to the unicode system that can't be displayed on the web).</p><p>Because of technical limitations, Yellow Lab Tools checks each font against the glyphs of the entire page. As a result, estimated use is >= to reality. For example, if you read that 10 glyphs are \"possibly used\", it means that these 10 glyphs are used on the page but nothing guaranties that they are displayed using this font.</p><p>Tools such as <a href=\"https://www.fontsquirrel.com/tools/webfont-generator\" target=\"_blank\">Font Squirrel</a> can remove some unicode ranges from a font.</p><p>In the case of an icon font, make sure you only keep the icons that are used on the website and to remove the others. Several tools are able to extract SVG images from a font, then some other tools can generate a font from the SVGs you want to keep.</p>",
+        "isOkThreshold": 0,
+        "isBadThreshold": 8,
+        "isAbnormalThreshold": 12,
+        "hasOffenders": true,
+        "offendersTransformFn": function(offenders) {
+            return offenders;
+        }
+    },
     "http2": {
         "label": "HTTP/2 or SPDY",
         "message": "<p>HTTP/2 is the latest version of the HTTP protocol and is designed to optimize load speed. SPDY is deprecated but still very well supported.</p><p>The latest versions of all major browsers are now compatible. The difficulty is on the server side, where technologies are not quite ready yet.</p>",

lib/metadata/scoreProfileGeneric.json

@@ -61,7 +61,6 @@
             "policies": {
                 "jQueryVersion": 2,
                 "jQueryVersionsLoaded": 2,
-                "jQueryFunctionsUsed": 1,
                 "jQueryCallsOnEmptyObject": 1,
                 "jQueryNotDelegatedEvents": 1
             }
@@ -93,6 +92,14 @@
                 "cssRedundantChildNodesSelectors": 1
             }
         },
+        "fonts": {
+            "label": "Web fonts",
+            "policies": {
+                "fontsCount": 1,
+                "heavyFonts": 1,
+                "unusedUnicodeRanges": 1
+            }
+        },
         "serverConfig": {
             "label": "Server config",
             "policies": {
@@ -115,6 +122,7 @@
         "cssSyntaxError": 1,
         "cssComplexity": 1,
         "badCSS": 1,
+        "fonts": 1,
         "serverConfig": 1
     }
 }

lib/offendersHelpers.js

@@ -175,7 +175,7 @@ var OffendersHelpers = function() {
         // Remove any line breaks
         offender = offender.replace(/(\r\n|\n|\r)/gm, '');
 
-        var parts = /^(.*) (?:<([^ \(]*)>|\[inline CSS\]) @ (\d+):(\d+)$/.exec(offender);
+        var parts = /^(.*) (?:<([^ \(]*)>|\[inline CSS\]) ?@ ?(\d+):(\d+)$/.exec(offender);
         
         if (!parts) {
             return {
@@ -207,24 +207,31 @@ var OffendersHelpers = function() {
     };
 
     this.orderByFile = function(offenders) {
-        var byFile = {};
+        var byFileObj = {};
 
         offenders.forEach(function(offender) {
             var file = offender.file || 'Inline CSS';
             delete offender.file;
 
-            if (!byFile[file]) {
-                byFile[file] = {
+            if (!byFileObj[file]) {
+                byFileObj[file] = {
+                    url: file,
                     count: 0,
                     offenders: []
                 };
             }
 
-            byFile[file].count ++;
-            byFile[file].offenders.push(offender);
+            byFileObj[file].count ++;
+            byFileObj[file].offenders.push(offender);
         });
 
-        return {byFile: byFile};
+        // Transform object into array
+        var byFileArray = [];
+        for (var file in byFileObj) {
+            byFileArray.push(byFileObj[file]);
+        }
+
+        return {byFile: byFileArray};
     };
 
 };

lib/screenshotHandler.js

@@ -1,5 +1,5 @@
 var debug       = require('debug')('ylt:screenshotHandler');
-var lwip        = require('lwip');
+var Jimp        = require('jimp');
 var tmp         = require('temporary');
 var Q           = require('q');
 var fs          = require('fs');
@@ -57,7 +57,7 @@ var screenshotHandler = function() {
     this.openImage = function(imagePath) {
         var deferred = Q.defer();
 
-        lwip.open(imagePath, function(err, image){
+        Jimp.read(imagePath, function(err, image){
             if (err) {
                 debug('Could not open imagePath %s', imagePath);
                 debug(err);
@@ -76,7 +76,7 @@ var screenshotHandler = function() {
     this.resizeImage = function(image, newWidth) {
         var deferred = Q.defer();
 
-        var currentWidth = image.width();
+        var currentWidth = image.bitmap.width;
         var ratio = newWidth / currentWidth;
 
         image.scale(ratio, function(err, image){
@@ -101,7 +101,7 @@ var screenshotHandler = function() {
         var deferred = Q.defer();
 
         // Create a canvas with the same dimensions as your image:
-        lwip.create(image.width(), image.height(), 'white', function(err, canvas){
+        new Jimp(image.bitmap.width, image.bitmap.height, 0xFFFFFF, function(err, canvas){
             if (err) {
                 debug('Could not create a white canvas');
                 debug(err);
@@ -109,7 +109,7 @@ var screenshotHandler = function() {
                 deferred.reject(err);
             } else {
                 // Paste original image on top of the canvas
-                canvas.paste(0, 0, image, function(err, image){
+                canvas.composite(image, 0, 0, function(err, image){
                     if (err) {
                         debug('Could not paste image on the white canvas');
                         debug(err);
@@ -131,7 +131,7 @@ var screenshotHandler = function() {
     this.toBuffer = function(image) {
         var deferred = Q.defer();
 
-        image.toBuffer('jpg', {quality: 90}, function(err, buffer){
+        image.quality(90).getBuffer(Jimp.MIME_JPEG, function(err, buffer){
             if (err) {
                 debug('Could not save image to buffer');
                 debug(err);
@@ -143,7 +143,7 @@ var screenshotHandler = function() {
             }
         });
 
-        return deferred.promise;        
+        return deferred.promise;
     };
 
 
@@ -167,4 +167,4 @@ var screenshotHandler = function() {
     };
 };
 
-module.exports = new screenshotHandler();
+module.exports = new screenshotHandler();

lib/server/controllers/apiController.js

@@ -6,7 +6,8 @@ var ScreenshotHandler   = require('../../screenshotHandler');
 var RunsQueue           = require('../datastores/runsQueue');
 var RunsDatastore       = require('../datastores/runsDatastore');
 var ResultsDatastore    = require('../datastores/resultsDatastore');
-var serverSettings      = require('../../../server_config/settings.json');
+
+var serverSettings      = (process.env.IS_TEST) ? require('../../../test/fixtures/settings.json') : require('../../../server_config/settings.json');
 
 var ApiController = function(app) {
     'use strict';
@@ -23,15 +24,23 @@ var ApiController = function(app) {
             req.body.url = 'http://' + req.body.url;
         }
 
+        // Block requests to unwanted websites (=spam)
+        if (req.body.url && isBlocked(req.body.url)) {
+            console.error('Test blocked for URL: %s', req.body.url);
+            res.status(403).send('Forbidden');
+            return;
+        }
+
         // Grab the test parameters and generate a random run ID
         var run = {
             runId: (Date.now()*1000 + Math.round(Math.random()*1000)).toString(36),
             params: {
                 url: req.body.url,
-                waitForResponse: req.body.waitForResponse !== false && req.body.waitForResponse !== 'false' && req.body.waitForResponse !== 0,
+                waitForResponse: req.body.waitForResponse === true || req.body.waitForResponse === 'true' || req.body.waitForResponse === 1,
                 partialResult: req.body.partialResult || null,
                 screenshot: req.body.screenshot || false,
                 device: req.body.device || 'desktop',
+                proxy: req.body.proxy || null,
                 waitForSelector: req.body.waitForSelector || null,
                 cookie: req.body.cookie || null,
                 authUser: req.body.authUser || null,
@@ -66,11 +75,12 @@ var ApiController = function(app) {
 
             runsDatastore.updatePosition(run.runId, 0);
 
-            debug('Launching test %s on %s', run.runId, run.params.url);
+            console.log('Launching test ' + run.runId + ' on ' + run.params.url);
 
             var runOptions = {
                 screenshot: run.params.screenshot ? screenshot.getTmpFilePath() : false,
                 device: run.params.device,
+                proxy: run.params.proxy,
                 waitForSelector: run.params.waitForSelector,
                 cookie: run.params.cookie,
                 authUser: run.params.authUser,
@@ -84,6 +94,7 @@ var ApiController = function(app) {
             return ylt(run.params.url, runOptions);
 
         })
+
         // Phantomas completed, let's save the screenshot if any
         .then(function(data) {
 
@@ -107,7 +118,7 @@ var ApiController = function(app) {
                             data.screenshotBuffer = screenshotBuffer;
 
                             // Official path to get the image
-                            data.screenshotUrl = '/api/results/' + data.runId + '/screenshot.jpg';
+                            data.screenshotUrl = 'api/results/' + data.runId + '/screenshot.jpg';
                         }
 
                     })
@@ -193,7 +204,7 @@ var ApiController = function(app) {
 
         // The user doesn't want to wait for the response, sending the run ID only
         if (!run.params.waitForResponse) {
-            console.log('Sending response without waiting.');
+            debug('Sending response without waiting.');
             res.setHeader('Content-Type', 'application/json');
             res.send(JSON.stringify({runId: run.runId}));
         }
@@ -325,6 +336,15 @@ var ApiController = function(app) {
             });
     });
 
+    function isBlocked(url) {
+        if (!serverSettings.blockedUrls) {
+            return false;
+        }
+
+        return serverSettings.blockedUrls.some(function(blockedUrl) {
+            return (url.indexOf(blockedUrl) === 0);
+        });
+    }
 };
 
 module.exports = ApiController;

lib/server/controllers/frontController.js

@@ -14,7 +14,7 @@ var FrontController = function(app) {
         app.get(route, function(req, res) {
             res.setHeader('Cache-Control', 'public, max-age=20');
             res.render(path.join(__dirname, assetsPath, 'main.html'), {
-                baseUrl: app.locals.baseUrl
+                baseUrl: app.locals.baseUrl || '/'
             });
         });
     });
@@ -23,7 +23,7 @@ var FrontController = function(app) {
     app.get('/views/:viewName', function(req, res) {
         res.setHeader('Cache-Control', 'public, max-age=' + cacheDuration);
         res.render(path.join(__dirname, assetsPath, 'views/' + req.params.viewName), {
-            baseUrl: app.locals.baseUrl
+            baseUrl: app.locals.baseUrl || '/'
         });
     });
     

lib/server/middlewares/apiLimitsMiddleware.js

@@ -1,6 +1,6 @@
-var config      = require('../../../server_config/settings.json');
+var config = (process.env.IS_TEST) ? require('../../../test/fixtures/settings.json') : require('../../../server_config/settings.json');
 
-var debug       = require('debug')('apiLimitsMiddleware');
+var debug = require('debug')('apiLimitsMiddleware');
 
 
 var apiLimitsMiddleware = function(req, res, next) {

lib/server/middlewares/authMiddleware.js

@@ -1,6 +1,6 @@
-var config      = require('../../../server_config/settings.json');
+var config = (process.env.IS_TEST) ? require('../../../test/fixtures/settings.json') : require('../../../server_config/settings.json');
 
-var debug       = require('debug')('authMiddleware');
+var debug = require('debug')('authMiddleware');
 
 
 var authMiddleware = function(req, res, next) {

lib/tools/jsExecutionTransformer.js

@@ -25,11 +25,8 @@ var jsExecutionTransformer = function() {
         var hasjQuery = (data.toolsResults.phantomas.metrics.jQueryVersionsLoaded > 0);
         if (hasjQuery) {
             metrics.jQueryCalls = 0;
-            metrics.jQueryFunctionsUsed = 0;
             metrics.jQueryCallsOnEmptyObject = 0;
             metrics.jQueryNotDelegatedEvents = 0;
-
-            offenders.jQueryFunctionsUsed = [];
         }
 
         try {
@@ -102,21 +99,6 @@ var jsExecutionTransformer = function() {
                     metrics.DOMaccesses += countTreeLeafs(node);
                 });
 
-                // Count the number of different jQuery functions called
-                if (hasjQuery) {
-
-                    jQueryFunctionsCollection.sort().forEach(function(fnName, cnt) {
-                        if (fnName === 'jQuery - find') {
-                            fnName = 'jQuery - $';
-                        }
-                        metrics.jQueryFunctionsUsed ++;
-                        offenders.jQueryFunctionsUsed.push({
-                            functionName: fnName.substring(9),
-                            count: cnt
-                        });
-                    });
-                }
-
             }
             debug('JS execution transformation complete');
 

lib/tools/phantomas/custom_modules/modules/charactersCount/charactersCount.js

@@ -0,0 +1,71 @@
+/**
+ * Lists the number of different characters
+ */
+/* global document: true, Node: true, window: true */
+
+exports.version = '1.0';
+
+exports.module = function(phantomas) {
+    'use strict';
+
+    //phantomas.setMetric('differentCharacters'); // @desc the number of different characters in the body @offenders
+
+    phantomas.on('report', function() {
+        phantomas.log("charactersCount: starting to analyze characters on page...");
+
+        phantomas.evaluate(function() {
+            (function(phantomas) {
+
+                phantomas.spyEnabled(false, 'analyzing characters on page');
+
+                function getLetterCount(arr){
+                    return arr.reduce(function(prev, next){
+                        prev[next] = 1;
+                        return prev;
+                    }, {});
+                }
+
+                if (document.body && document.body.textContent) {
+                    var allText = '';
+
+                    // Traverse all DOM Tree to read text
+                    var runner = new phantomas.nodeRunner();
+                    runner.walk(document.body, function(node, depth) {
+                        switch (node.nodeType) {
+
+                            // Grabing text nodes
+                            case Node.TEXT_NODE:
+                                if (node.parentNode.tagName !== 'SCRIPT' && node.parentNode.tagName !== 'STYLE') {
+                                    allText += node.nodeValue;
+                                }
+                                break;
+
+                            // Grabing CSS content properties
+                            case Node.ELEMENT_NODE:
+                                if (node.tagName !== 'SCRIPT' && node.tagName !== 'STYLE') {
+                                    allText += window.getComputedStyle(node).getPropertyValue('content');
+                                    allText += window.getComputedStyle(node, ':before').getPropertyValue('content');
+                                    allText += window.getComputedStyle(node, ':after').getPropertyValue('content');
+                                }
+                                break;
+                        }
+                    });
+
+                    // Reduce all text found in page into a string of unique chars
+                    var charactersList = getLetterCount(allText.split(''));
+                    var charsetString = Object.keys(charactersList).sort().join('');
+                    
+                    // Remove blank characters
+                    charsetString = charsetString.replace(/\s/g, '');
+
+                    phantomas.setMetric('differentCharacters', charsetString.length);
+                    phantomas.addOffender('differentCharacters', charsetString);
+                }
+
+                phantomas.spyEnabled(true);
+            }(window.__phantomas));
+        });
+
+        phantomas.log("charactersCount: analyzing characters done.");
+    });
+};

lib/tools/phantomas/custom_modules/modules/domQYLT/domQYLT.js

@@ -39,9 +39,9 @@ exports.module = function(phantomas) {
                     });
 
                 }, function(result, args) {
-                    var id = args[0];
+                    var id = args ? '#' + args[0] : undefined;
 
-                    querySpy('id', '#' + id, 'getElementById', '#document', (result === null));
+                    querySpy('id', id, 'getElementById', '#document', (result === null));
 
                     var moreData = {
                         resultsNumber : (result === null) ? 0 : 1
@@ -74,10 +74,10 @@ exports.module = function(phantomas) {
                 function selectorClassNameAfter(result, args) {
                     /*jshint validthis: true */
 
-                    var className = args[0];
+                    var className = args ? '.' + args[0] : undefined;
                     var context = phantomas.getDOMPath(this);
 
-                    querySpy('class', '.' + className, 'getElementsByClassName', context, (result.length === 0));
+                    querySpy('class', className, 'getElementsByClassName', context, (result.length === 0));
                     
                     var moreData = {
                         resultsNumber : (result && result.length > 0) ? result.length : 0
@@ -113,10 +113,10 @@ exports.module = function(phantomas) {
                 function selectorTagNameSpyAfter(result, args) {
                     /*jshint validthis: true */
                     
-                    var tagName = args[0];
+                    var tagName = args ? args[0].toLowerCase() : undefined;
                     var context = phantomas.getDOMPath(this);
 
-                    querySpy('tag name', tagName.toLowerCase(), 'getElementsByTagName', context, (result.length === 0));
+                    querySpy('tag name', tagName, 'getElementsByTagName', context, (result.length === 0));
                     
                     var moreData = {
                         resultsNumber : (result && result.length > 0) ? result.length : 0
@@ -156,7 +156,7 @@ exports.module = function(phantomas) {
                 function selectorQuerySpyAfter(result, args) {
                     /*jshint validthis: true */
 
-                    var selector = args[0];
+                    var selector = args ? args[0] : undefined;
                     var context = phantomas.getDOMPath(this);
 
                     querySpy('selector', selector, 'querySelectorAll', context, (!result || result.length === 0));
@@ -189,7 +189,7 @@ exports.module = function(phantomas) {
                 function selectorAllQuerySpryAfter(result, args) {
                     /*jshint validthis: true */
 
-                    var selector = args[0];
+                    var selector = args ? args[0] : undefined;
                     var context = phantomas.getDOMPath(this);
 
                     querySpy('selector', selector, 'querySelectorAll', context, (!result || result.length === 0));

lib/tools/phantomas/custom_modules/modules/lazyLoadableYLT/lazyLoadableYLT.js

@@ -26,7 +26,9 @@ exports.module = function(phantomas) {
                     path,
                     processedImages = {},
                     src,
-                    viewportHeight = window.innerHeight;
+                    viewportHeight = window.innerHeight,
+                    // Add an offset of 100px under the height of the screen
+                    LAZYLOAD_OFFSET = 100;
 
                 phantomas.log('lazyLoadableImages: %d image(s) found, assuming %dpx offset to be the fold', len, viewportHeight);
 
@@ -64,7 +66,7 @@ exports.module = function(phantomas) {
                 Object.keys(processedImages).forEach(function(src) {
                     var img = processedImages[src];
 
-                    if (img.offset > viewportHeight) {
+                    if (img.offset > viewportHeight + LAZYLOAD_OFFSET) {
                         phantomas.log('lazyLoadableImages: <%s> image (%s) is below the fold (at %dpx)', src, img.path, img.offset);
 
                         phantomas.incrMetric('lazyLoadableImagesBelowTheFold');

lib/tools/phantomas/phantomasWrapper.js

@@ -59,6 +59,12 @@ var PhantomasWrapper = function() {
             ].join(',')
         };
 
+        // Proxy option can't be set to null or undefined...
+        // this is why it's set now and not in the object above
+        if (task.options.proxy) {
+            options.proxy = task.options.proxy;
+        }
+
         // Output the command line for debugging purpose
         debug('If you want to reproduce the phantomas task only, copy the following command line:');
         var optionsString = '';

lib/tools/redownload/contentTypeChecker.js

@@ -18,11 +18,11 @@ var ContentTypeChecker = function() {
         debug('Entering contentTypeChecker');
         
         // Ignore very small files as they are generally tracking pixels
-        if (entry.weightCheck && entry.weightCheck.body && entry.weightCheck.bodySize > 100) {
+        if (entry.weightCheck && entry.weightCheck.bodyBuffer && entry.weightCheck.bodySize > 100) {
             var foundType;
 
             try {
-                foundType = findContentType(entry.weightCheck.body);
+                foundType = findContentType(entry.weightCheck.bodyBuffer);
             
                 if (!entry.contentType || entry.contentType === '') {
                     if (foundType === null) {
@@ -51,43 +51,43 @@ var ContentTypeChecker = function() {
         return deferred.promise;
     }
 
-    function findContentType(body) {
-        var buffer = new Buffer(body, 'binary');
+    function findContentType(bodyBuffer) {
+        var bodyStr = bodyBuffer.toString();
 
-        if (isJpg(buffer)) {
+        if (isJpg(bodyBuffer)) {
             return contentTypes.jpeg;
         }
 
-        if (isPng(buffer)) {
+        if (isPng(bodyBuffer)) {
             return contentTypes.png;
         }
 
         // https://github.com/sindresorhus/is-svg/issues/7
-        if (/<svg/.test(body) && isSvg(body)) {
+        if (/<svg/.test(bodyStr) && isSvg(bodyStr)) {
             return contentTypes.svg;
         }
 
-        if (isGif(buffer)) {
+        if (isGif(bodyBuffer)) {
             return contentTypes.gif;
         }
 
-        if (isWoff(buffer)) {
+        if (isWoff(bodyBuffer)) {
             return contentTypes.woff;
         }
 
-        if (isWoff2(buffer)) {
+        if (isWoff2(bodyBuffer)) {
             return contentTypes.woff2;
         }
 
-        if (isOtf(buffer)) {
+        if (isOtf(bodyBuffer)) {
             return contentTypes.otf;
         }
 
-        if (isTtf(buffer)) {
+        if (isTtf(bodyBuffer)) {
             return contentTypes.ttf;
         }
 
-        if (isEot(buffer)) {
+        if (isEot(bodyBuffer)) {
             return contentTypes.eot;
         }
 

lib/tools/redownload/fileMinifier.js

@@ -11,22 +11,24 @@ var FileMinifier = function() {
     function minifyFile(entry) {
         var deferred = Q.defer();
 
-        if (!entry.weightCheck || !entry.weightCheck.body) {
+        if (!entry.weightCheck || !entry.weightCheck.bodyBuffer) {
             // No valid file available
             deferred.resolve(entry);
             return deferred.promise;
         }
 
         var fileSize = entry.weightCheck.uncompressedSize;
+        var bodyString = entry.weightCheck.bodyBuffer.toString();
+
         debug('Let\'s try to optimize %s', entry.url);
         debug('Current file size is %d', fileSize);
         var startTime = Date.now();
 
-        if (entry.isJS && !isKnownAsMinified(entry.url) && !looksAlreadyMinified(entry.weightCheck.body)) {
+        if (entry.isJS && !isKnownAsMinified(entry.url) && !looksAlreadyMinified(bodyString)) {
 
             debug('File is a JS');
 
-            return minifyJs(entry.weightCheck.body)
+            return minifyJs(bodyString)
 
             .then(function(newFile) {
                 if (!newFile) {
@@ -58,7 +60,7 @@ var FileMinifier = function() {
 
             debug('File is a CSS');
 
-            return minifyCss(entry.weightCheck.body)
+            return minifyCss(entry.weightCheck.bodyBuffer.toString())
 
             .then(function(newFile) {
                 if (!newFile) {
@@ -91,7 +93,7 @@ var FileMinifier = function() {
 
             debug('File is an HTML');
 
-            return minifyHtml(entry.weightCheck.body)
+            return minifyHtml(entry.weightCheck.bodyBuffer.toString())
 
             .then(function(newFile) {
                 if (!newFile) {
@@ -294,7 +296,7 @@ var FileMinifier = function() {
         return deferred.promise;
     }
 
-    // Avoid loosing time trying to compress some JS libraries known as already compressed
+    // Avoid losing time trying to compress some JS libraries known as already compressed
     function isKnownAsMinified(url) {
         var result = false;
 
@@ -320,7 +322,7 @@ var FileMinifier = function() {
         return result;
     }
 
-    // Avoid loosing some trying to compress JS files if they alreay look minified
+    // Avoid losing some trying to compress JS files if they already look minified
     // by counting the number of lines compared to the total size.
     // Less than 2KB per line is suspicious
     function looksAlreadyMinified(code) {
@@ -349,4 +351,4 @@ var FileMinifier = function() {
     };
 };
 
-module.exports = new FileMinifier();
+module.exports = new FileMinifier();

lib/tools/redownload/fontAnalyzer.js

@@ -0,0 +1,388 @@
+var debug = require('debug')('ylt:fontAnalyzer');
+
+var Q           = require('q');
+var fontkit     = require('fontkit');
+
+var FontAnalyzer = function() {
+
+    function analyzeFont(entry, charsListOnPage) {
+        var deferred = Q.defer();
+        
+        if (!entry.weightCheck || !entry.weightCheck.bodyBuffer) {
+            // No valid file available
+            deferred.resolve(entry);
+            return deferred.promise;
+        }
+
+        var fileSize = entry.weightCheck.uncompressedSize;
+
+        if (entry.isWebFont) {
+            debug('File %s is a font. Let\'s have a look inside!', entry.url);
+            
+            getMetricsFromFont(entry, charsListOnPage)
+
+            .then(function(fontMetrics) {
+                entry.fontMetrics = fontMetrics;
+                deferred.resolve(entry);
+            })
+
+            .fail(function(error) {
+                debug('Could not open the font: %s', error);
+                deferred.resolve(entry);
+            });
+
+        } else {
+            deferred.resolve(entry);
+        }
+        
+        return deferred.promise;
+    }
+
+    function getMetricsFromFont(entry, charsListOnPage) {
+        var deferred = Q.defer();
+        
+        try {
+            var startTime = Date.now();
+            var font = fontkit.create(entry.weightCheck.bodyBuffer);
+
+            var result = {
+                name: font.fullName || font.postscriptName || font.familyName,
+                numGlyphs: font.numGlyphs,
+                averageGlyphComplexity: getAverageGlyphComplexity(font),
+                compressedWeight: entry.weightCheck.afterCompression || entry.weightCheck.bodySize,
+                unicodeRanges: readUnicodeRanges(font.characterSet, charsListOnPage),
+                numGlyphsInCommonWithPageContent: countPossiblyUsedGlyphs(getCharacterSetAsString(font.characterSet), charsListOnPage)
+            };
+
+            var endTime = Date.now();
+            debug('Font analysis took %dms', endTime - startTime);
+
+            // Mark fonts that are not used on the page (#224)
+            var fontIsUsed = false;
+            for (var range in result.unicodeRanges) {
+                if (result.unicodeRanges[range].numGlyphsInCommonWithPageContent > 0) {
+                    fontIsUsed = true;
+                    break;
+                }
+            }
+            result.isUsed = fontIsUsed;
+
+            deferred.resolve(result);
+        } catch(error) {
+            deferred.reject(error);
+        }
+
+        return deferred.promise;
+    }
+
+    // Reads the number of vector commands (complexity) needed to render glyphs and
+    // returns the average. Only first 100 glyphes are tested, otherwise it would take tool long;
+    function getAverageGlyphComplexity(font) {
+        var max = Math.min(font.numGlyphs, 100);
+
+        var totalPathsCommands = 0;
+        for (var i = 0; i < max; i++) {
+            totalPathsCommands += font.getGlyph(i).path.commands.length;
+        }
+        return Math.round(totalPathsCommands / max * 10) / 10;
+    }
+
+    function readUnicodeRanges(charsetInFont, charsListOnPage) {
+        var ranges = {};
+
+        // Assign a range to each char found in the font
+        charsetInFont.forEach(function(char) {
+            
+            var currentRange = getUnicodeRangeFromChar(char);
+            var currentRangeName = currentRange.name;
+
+            if (!ranges[currentRangeName]) {
+                // Cloning the object
+                ranges[currentRangeName] = Object.assign({}, currentRange);
+            }
+
+            if (!ranges[currentRangeName].charset) {
+                ranges[currentRangeName].charset = '';
+            }
+
+            ranges[currentRangeName].charset += String.fromCharCode(char);
+        });
+
+        var range;
+        var expectedLength;
+        var actualLength;
+
+        for (var rangeName in ranges) {
+            /*jshint loopfunc: true */
+
+            range = ranges[rangeName];
+
+            // Estimate if range is used, based on the characters found in the page
+            range.numGlyphsInCommonWithPageContent = countPossiblyUsedGlyphs(range.charset, charsListOnPage);
+
+            // Calculate coverage
+            if (rangeName !== 'Others') {
+                expectedLength = range.rangeEnd - range.rangeStart + 1;
+                actualLength = range.charset.length;
+                range.coverage = Math.min(actualLength / expectedLength, 1);
+            }
+        }
+
+        return ranges;
+    }
+
+    function countPossiblyUsedGlyphs(charsetInFont, charsListOnPage) {
+        var count = 0;
+        charsListOnPage.split('').forEach(function(char) {
+            if (charsetInFont.indexOf(char) >= 0) {
+                count ++;
+            }
+        });
+        return count;
+    }
+
+    function getCharacterSetAsString(characterSet) {
+        var str = '';
+        characterSet.forEach(function(charCode) {
+            str += String.fromCharCode(charCode);
+        });
+        return str;
+    }
+
+    function getUnicodeRangeFromChar(char) {
+        return UNICODE_RANGES.find(function(range) {
+            return (char >= range.rangeStart && char <= range.rangeEnd);
+        }) || {name: 'Others'};
+    }
+
+
+    var UNICODE_RANGES = [
+        {
+            name: 'Basic Latin',
+            rangeStart: 0x0020,
+            rangeEnd: 0x007F
+        },
+        {
+            name: 'Latin-1 Supplement',
+            rangeStart: 0x00A0,
+            rangeEnd: 0x00FF
+        },
+        {
+            name: 'Latin Extended',
+            rangeStart: 0x0100,
+            rangeEnd: 0x024F
+        },
+        {
+            name: 'IPA Extensions',
+            rangeStart: 0x0250,
+            rangeEnd: 0x02AF
+        },
+        {
+            name: 'Greek and Coptic',
+            rangeStart: 0x0370,
+            rangeEnd: 0x03FF
+        },
+        {
+            name: 'Cyrillic',
+            rangeStart: 0x0400,
+            rangeEnd: 0x052F
+        },
+        {
+            name: 'Armenian',
+            rangeStart: 0x0530,
+            rangeEnd: 0x058F
+        },
+        {
+            name: 'Hebrew',
+            rangeStart: 0x0590,
+            rangeEnd: 0x05FF
+        },
+        {
+            name: 'Arabic',
+            rangeStart: 0x0600,
+            rangeEnd: 0x06FF
+        },
+        {
+            name: 'Syriac',
+            rangeStart: 0x0700,
+            rangeEnd: 0x074F
+        },
+        {
+            name: 'Thaana',
+            rangeStart: 0x0780,
+            rangeEnd: 0x07BF
+        },
+        {
+            name: 'Devanagari',
+            rangeStart: 0x0900,
+            rangeEnd: 0x097F
+        },
+        {
+            name: 'Bengali',
+            rangeStart: 0x0980,
+            rangeEnd: 0x09FF
+        },
+        {
+            name: 'Gurmukhi',
+            rangeStart: 0x0A00,
+            rangeEnd: 0x0A7F
+        },
+        {
+            name: 'Gujarati',
+            rangeStart: 0x0A80,
+            rangeEnd: 0x0AFF
+        },
+        {
+            name: 'Oriya',
+            rangeStart: 0x0B00,
+            rangeEnd: 0x0B7F
+        },
+        {
+            name: 'Tamil',
+            rangeStart: 0x0B80,
+            rangeEnd: 0x0BFF
+        },
+        {
+            name: 'Telugu',
+            rangeStart: 0x0C00,
+            rangeEnd: 0x0C7F
+        },
+        {
+            name: 'Kannada',
+            rangeStart: 0x0C80,
+            rangeEnd: 0x0CFF
+        },
+        {
+            name: 'Malayalam',
+            rangeStart: 0x0D00,
+            rangeEnd: 0x0D7F
+        },
+        {
+            name: 'Sinhala',
+            rangeStart: 0x0D80,
+            rangeEnd: 0x0DFF
+        },
+        {
+            name: 'Thai',
+            rangeStart: 0x0E00,
+            rangeEnd: 0x0E7F
+        },
+        {
+            name: 'Lao',
+            rangeStart: 0x0E80,
+            rangeEnd: 0x0EFF
+        },
+        {
+            name: 'Tibetan',
+            rangeStart: 0x0F00,
+            rangeEnd: 0x0FFF
+        },
+        {
+            name: 'Myanmar',
+            rangeStart: 0x1000,
+            rangeEnd: 0x109F
+        },
+        {
+            name: 'Georgian',
+            rangeStart: 0x10A0,
+            rangeEnd: 0x10FF
+        },
+        {
+            name: 'Hangul Jamo',
+            rangeStart: 0x1100,
+            rangeEnd: 0x11FF
+        },
+        {
+            name: 'Ethiopic',
+            rangeStart: 0x1200,
+            rangeEnd: 0x137F
+        },
+        {
+            name: 'Cherokee',
+            rangeStart: 0x13A0,
+            rangeEnd: 0x13FF
+        },
+        {
+            name: 'Unified Canadian Aboriginal Syllabics',
+            rangeStart: 0x1400,
+            rangeEnd: 0x167F
+        },
+        {
+            name: 'Ogham',
+            rangeStart: 0x1680,
+            rangeEnd: 0x169F
+        },
+        {
+            name: 'Runic',
+            rangeStart: 0x16A0,
+            rangeEnd: 0x16FF
+        },
+        {
+            name: 'Tagalog',
+            rangeStart: 0x1700,
+            rangeEnd: 0x171F
+        },
+        {
+            name: 'Hanunoo',
+            rangeStart: 0x1720,
+            rangeEnd: 0x173F
+        },
+        {
+            name: 'Buhid',
+            rangeStart: 0x1740,
+            rangeEnd: 0x175F
+        },
+        {
+            name: 'Tagbanwa',
+            rangeStart: 0x1760,
+            rangeEnd: 0x177F
+        },
+        {
+            name: 'Khmer',
+            rangeStart: 0x1780,
+            rangeEnd: 0x17FF
+        },
+        {
+            name: 'Mongolian',
+            rangeStart: 0x1800,
+            rangeEnd: 0x18AF
+        },
+        {
+            name: 'Limbu',
+            rangeStart: 0x1900,
+            rangeEnd: 0x194F
+        },
+        {
+            name: 'Tai Le',
+            rangeStart: 0x1950,
+            rangeEnd: 0x197F
+        },
+        {
+            name: 'Hiragana',
+            rangeStart: 0x3040,
+            rangeEnd: 0x309F
+        },
+        {
+            name: 'Katakana',
+            rangeStart: 0x30A0,
+            rangeEnd: 0x30FF
+        },
+        {
+            name: 'Bopomofo',
+            rangeStart: 0x3100,
+            rangeEnd: 0x312F
+        }
+    ];
+
+    return {
+        analyzeFont: analyzeFont,
+        getMetricsFromFont: getMetricsFromFont,
+        readUnicodeRanges: readUnicodeRanges,
+        getAverageGlyphComplexity: getAverageGlyphComplexity,
+        countPossiblyUsedGlyphs: countPossiblyUsedGlyphs,
+        getCharacterSetAsString: getCharacterSetAsString,
+        getUnicodeRangeFromChar: getUnicodeRangeFromChar
+    };
+};
+
+module.exports = new FontAnalyzer();

lib/tools/redownload/gzipCompressor.js

@@ -16,12 +16,12 @@ var GzipCompressor = function() {
     function gzipUncompressedFile(entry) {
         var deferred = Q.defer();
 
-        if (entryTypeCanBeGzipped(entry) && entry.weightCheck && !entry.weightCheck.isCompressed && entry.weightCheck.body) {
+        if (entryTypeCanBeGzipped(entry) && entry.weightCheck && !entry.weightCheck.isCompressed && entry.weightCheck.bodyBuffer) {
             debug('Compression missing, trying to gzip file %s', entry.url);
 
             var uncompressedSize = entry.weightCheck.uncompressedSize;
 
-            zlib.gzip(new Buffer(entry.weightCheck.body, 'utf8'), function(err, buffer) {
+            zlib.gzip(entry.weightCheck.bodyBuffer, function(err, buffer) {
                 if (err) {
                     debug('Could not compress uncompressed file with gzip');
                     debug(err);

lib/tools/redownload/imageOptimizer.js

@@ -14,7 +14,7 @@ var ImageOptimizer = function() {
     function optimizeImage(entry) {
         var deferred = Q.defer();
 
-        if (!entry.weightCheck || !entry.weightCheck.body) {
+        if (!entry.weightCheck || !entry.weightCheck.bodyBuffer) {
             // No valid file available
             deferred.resolve(entry);
             return deferred.promise;
@@ -28,7 +28,7 @@ var ImageOptimizer = function() {
             debug('File is a JPEG');
 
             // Starting softly with a lossless compression
-            return compressJpegLosslessly(new Buffer(entry.weightCheck.body, 'binary'))
+            return compressJpegLosslessly(entry.weightCheck.bodyBuffer)
 
             .then(function(newFile) {
                 if (!newFile) {
@@ -48,7 +48,7 @@ var ImageOptimizer = function() {
 
 
                 // Now let's compress lossy to MAX_JPEG_QUALITY
-                return compressJpegLossly(new Buffer(entry.weightCheck.body, 'binary'));
+                return compressJpegLossly(entry.weightCheck.bodyBuffer);
             })
             
             .then(function(newFile) {
@@ -86,7 +86,7 @@ var ImageOptimizer = function() {
             debug('File is a PNG');
 
             // Starting softly with a lossless compression
-            return compressPngLosslessly(new Buffer(entry.weightCheck.body, 'binary'))
+            return compressPngLosslessly(entry.weightCheck.bodyBuffer)
 
             .then(function(newFile) {
                 if (!newFile) {
@@ -120,7 +120,7 @@ var ImageOptimizer = function() {
             debug('File is an SVG');
 
             // Starting softly with a lossless compression
-            return compressSvgLosslessly(new Buffer(entry.weightCheck.body, 'utf8'))
+            return compressSvgLosslessly(entry.weightCheck.bodyBuffer)
 
             .then(function(newFile) {
                 if (!newFile) {

lib/tools/redownload/redownload.js

@@ -18,6 +18,7 @@ var imageOptimizer      = require('./imageOptimizer');
 var fileMinifier        = require('./fileMinifier');
 var gzipCompressor      = require('./gzipCompressor');
 var contentTypeChecker  = require('./contentTypeChecker');
+var fontAnalyzer        = require('./fontAnalyzer');
 
 
 var Redownload = function() {
@@ -44,11 +45,25 @@ var Redownload = function() {
             };
         }
 
+        var proxy = null;
+        if (data.params && data.params.options && data.params.options.proxy) {
+            proxy = data.params.options.proxy;
+            if (proxy.indexOf('http:') === -1) {
+                proxy = 'http://' + proxy;
+            }
+        }
+
+        // Prevent a bug with the font analyzer on empty pages
+        var differentCharacters = '';
+        if (data.toolsResults.phantomas.offenders.differentCharacters && data.toolsResults.phantomas.offenders.differentCharacters.length > 0) {
+            differentCharacters = data.toolsResults.phantomas.offenders.differentCharacters[0];
+        }
+
         // Transform every request into a download function with a callback when done
         var redownloadList = requestsList.map(function(entry) {
             return function(callback) {
                 
-                redownloadEntry(entry, httpAuth)
+                redownloadEntry(entry, httpAuth, proxy)
 
                 .then(contentTypeChecker.checkContentType)
 
@@ -58,8 +73,12 @@ var Redownload = function() {
 
                 .then(gzipCompressor.compressFile)
 
+                .then(function(entry) {
+                    return fontAnalyzer.analyzeFont(entry, differentCharacters);
+                })
+
                 .then(function(newEntry) {
-                    debug('File %s - Redownloaded, optimized, minified, compressed: done', entry.url);
+                    debug('File %s - Redownloaded, optimized, minified, compressed, analyzed: done', entry.url);
                     callback(null, newEntry);
                 })
 
@@ -84,6 +103,14 @@ var Redownload = function() {
                 var metrics = {};
                 var offenders = {};
 
+                // Remove unused fonts that a normal browser would not download (fix #224)
+                results = results.filter(function(result) {
+                    if (result && result.fontMetrics) {
+                        return result.fontMetrics.isUsed !== false;
+                    }
+                    return true;
+                });
+
                 // Count requests
                 offenders.totalRequests = listRequestsByType(results);
                 metrics.totalRequests = offenders.totalRequests.total;
@@ -134,6 +161,18 @@ var Redownload = function() {
                 offenders.identicalFiles = listIdenticalFiles(results);
                 metrics.identicalFiles = offenders.identicalFiles.avoidableRequests;
 
+                // Fonts count
+                offenders.fontsCount = listFonts(results);
+                metrics.fontsCount = offenders.fontsCount.count;
+
+                // Heavy fonts
+                offenders.heavyFonts = listHeavyFonts(results);
+                metrics.heavyFonts = offenders.heavyFonts.totalGain;
+
+                // Unused Unicode ranges
+                offenders.unusedUnicodeRanges = listUnusedUnicodeRanges(results);
+                metrics.unusedUnicodeRanges = offenders.unusedUnicodeRanges.count;
+
 
                 data.toolsResults.redownload = {
                     metrics: metrics,
@@ -381,7 +420,7 @@ var Redownload = function() {
         var avoidableRequestsCount = 0;
 
         requests.forEach(function(req) {
-            var requestHash = md5(req.weightCheck.body);
+            var requestHash = md5(req.weightCheck.bodyBuffer);
             
             // Try to exclude tracking pixels
             if (req.weightCheck.bodySize < 80 && req.type === 'image') {
@@ -414,8 +453,140 @@ var Redownload = function() {
         };
     }
 
+    function listFonts(requests) {
+        var list = [];
 
-    function redownloadEntry(entry, httpAuth) {
+        requests.forEach(function(req) {
+            if (req.isWebFont) {
+                list.push({
+                    url: req.url,
+                    size: req.weightCheck.bodySize
+                });
+            }
+        });
+
+        return {
+            count: list.length,
+            list: list
+        };
+    }
+
+    function listHeavyFonts(requests) {
+        var list = [];
+        var totalGain = 0;
+        var heavyFontsCount = 0;
+        var MAX_FONT_WEIGHT = 40 * 1024;
+
+        requests.forEach(function(req) {
+            if (req.isWebFont && req.fontMetrics) {
+                list.push({
+                    url: req.url,
+                    weight: req.weightCheck.bodySize,
+                    numGlyphs: req.fontMetrics.numGlyphs,
+                    averageGlyphComplexity: req.fontMetrics.averageGlyphComplexity
+                });
+                if (req.weightCheck.bodySize > MAX_FONT_WEIGHT) {
+                    totalGain += req.weightCheck.bodySize - MAX_FONT_WEIGHT;
+                    heavyFontsCount ++;
+                }
+            }
+        });
+
+        return {
+            count: heavyFontsCount,
+            fonts: list,
+            totalGain: totalGain
+        };
+    }
+
+    function listUnusedUnicodeRanges(requests) {
+        var list = [];
+        var unusedUnicodeRanges = 0;
+
+        requests.forEach(function(req) {
+            if (req.isWebFont && req.fontMetrics && req.fontMetrics.unicodeRanges) {
+                var ranges = [];
+                var others = null;
+                var rangeNames = Object.keys(req.fontMetrics.unicodeRanges);
+                var unicodePointsCount = 0;
+                var unusedRangesInFont = 0;
+                
+                rangeNames.forEach(function(rangeName) {
+                    var range = req.fontMetrics.unicodeRanges[rangeName];
+
+                    // Exclude "Others"
+                    if (rangeName === 'Others') {
+                        if (range.numGlyphsInCommonWithPageContent === 0 && range.charset.length > 50) {
+                            range.underused = true;
+                            unusedRangesInFont ++;
+                        }
+
+                        unicodePointsCount += range.charset.length;
+                        others = range;
+                    } else if (range.charset.length > 0) {
+                        // Now lets detect if the current Unicode range is unused.
+                        // Reminder: range.coverage = glyphs declared in this range, divided by the range size
+                        if (range.coverage > 0.25 && range.numGlyphsInCommonWithPageContent === 0) {
+                            range.underused = true;
+                            unusedRangesInFont ++;
+                        }
+
+                        unicodePointsCount += range.charset.length;
+                        ranges.push(range);
+                    }
+                });
+
+
+                // Detect if it's a icons font : if more than 90% of the icons are
+                // in the "Others", it looks like one.
+                if (others && others.charset.length / req.fontMetrics.numGlyphs > 0.9) {
+                    
+                    list.push({
+                        url: req.url,
+                        weight: req.weightCheck.bodySize,
+                        isIconFont: true,
+                        glyphs: req.fontMetrics.numGlyphs,
+                        numGlyphsInCommonWithPageContent: req.fontMetrics.numGlyphsInCommonWithPageContent
+                    });
+
+                    // And if less than 5% of the icons are used, let's report it as underused
+                    if (others && others.numGlyphsInCommonWithPageContent / others.charset.length <= 0.05) {
+                        unusedRangesInFont = 1;
+                    }
+
+                // Not an icons font
+                } else {
+                    if (others) {
+                        // Insert back "Others" at the end of the list
+                        ranges.push(others);
+                    }
+
+                    var ligaturesOrHiddenChars = req.fontMetrics.numGlyphs - unicodePointsCount;
+                    if (ligaturesOrHiddenChars > 25) {
+                        unusedUnicodeRanges ++;
+                    }
+
+                    list.push({
+                        url: req.url,
+                        weight: req.weightCheck.bodySize,
+                        isIconFont: false,
+                        unicodeRanges: ranges,
+                        ligaturesOrHiddenChars: ligaturesOrHiddenChars 
+                    });
+                }
+
+                unusedUnicodeRanges += unusedRangesInFont;
+            }
+        });
+
+        return {
+            count: unusedUnicodeRanges,
+            fonts: list
+        };
+    }
+
+
+    function redownloadEntry(entry, httpAuth, proxy) {
         var deferred = Q.defer();
         
         function downloadError(message) {
@@ -467,7 +638,8 @@ var Redownload = function() {
             method: entry.method,
             url: entry.url,
             headers: reqHeaders,
-            timeout: REQUEST_TIMEOUT
+            timeout: REQUEST_TIMEOUT,
+            proxy: proxy
         };
 
         // Basic auth
@@ -521,7 +693,7 @@ var Redownload = function() {
 
                 var uncompressedSize = 0;  // size after uncompression
                 var bodySize = 0;  // bytes size over the wire
-                var body = '';  // plain text body (after uncompressing gzip/deflate)
+                var bodyChunks = [];  // an array of buffers
                 var isCompressed = false;
 
                 function tally() {
@@ -531,8 +703,10 @@ var Redownload = function() {
                         return;
                     }
 
+                    var body = Buffer.concat(bodyChunks);
+
                     var result = {
-                        body: body,
+                        bodyBuffer: body,
                         headersSize: Buffer.byteLength(rawHeaders, 'utf8'),
                         bodySize: bodySize,
                         isCompressed: isCompressed,
@@ -548,7 +722,8 @@ var Redownload = function() {
                         var gzip = zlib.createGunzip();
 
                         gzip.on('data', function (data) {
-                            body += data;
+
+                            bodyChunks.push(data);
                             uncompressedSize += data.length;
                         }).on('end', function () {
                             isCompressed = true;
@@ -570,7 +745,7 @@ var Redownload = function() {
                         var deflate = zlib.createInflate();
 
                         deflate.on('data', function (data) {
-                            body += data;
+                            bodyChunks.push(data);
                             uncompressedSize += data.length;
                         }).on('end', function () {
                             isCompressed = true;
@@ -587,12 +762,8 @@ var Redownload = function() {
 
                         break;
                     default:
-                        if (contentType === 'image/jpeg' || contentType === 'image/png') {
-                            res.setEncoding('binary');
-                        }
-
                         res.on('data', function (data) {
-                            body += data;
+                            bodyChunks.push(data);
                             uncompressedSize += data.length;
                             bodySize += data.length;
                         }).on('end', function () {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "yellowlabtools",
-  "version": "1.11.1",
+  "version": "1.13.0",
   "description": "Online tool to audit a webpage for performance and front-end quality issues",
   "license": "GPL-2.0",
   "author": {
@@ -20,79 +20,80 @@
   },
   "main": "./lib/index.js",
   "dependencies": {
-    "angular": "1.5.7",
-    "angular-animate": "1.5.7",
-    "angular-chart.js": "0.10.2",
-    "angular-local-storage": "0.2.7",
-    "angular-resource": "1.5.7",
-    "angular-route": "1.5.7",
-    "angular-sanitize": "1.5.7",
-    "async": "1.5.2",
-    "body-parser": "1.15.2",
-    "chart.js": "1.1.1",
-    "clean-css": "3.4.18",
-    "color-diff": "1.0.0",
-    "compression": "1.6.2",
-    "cors": "2.7.1",
+    "angular": "1.6.8",
+    "angular-animate": "1.6.8",
+    "angular-chart.js": "1.1.1",
+    "angular-local-storage": "0.7.1",
+    "angular-resource": "1.6.8",
+    "angular-route": "1.6.8",
+    "angular-sanitize": "1.6.8",
+    "async": "2.6.0",
+    "body-parser": "1.18.2",
+    "chart.js": "2.7.1",
+    "clean-css": "4.1.9",
+    "color-diff": "1.1.0",
+    "compression": "1.7.1",
+    "cors": "2.8.4",
     "css-mq-parser": "0.0.3",
-    "debug": "2.2.0",
+    "debug": "3.1.0",
     "easyxml": "2.0.1",
-    "ejs": "^2.5.1",
-    "express": "4.14.0",
-    "imagemin": "5.2.2",
-    "imagemin-jpegoptim": "5.0.0",
+    "ejs": "2.5.7",
+    "express": "4.16.2",
+    "fontkit": "1.7.7",
+    "imagemin": "5.3.1",
+    "imagemin-jpegoptim": "5.2.0",
     "imagemin-jpegtran": "5.0.2",
-    "imagemin-optipng": "5.1.0",
-    "imagemin-svgo": "5.1.0",
+    "imagemin-optipng": "5.2.1",
+    "imagemin-svgo": "6.0.0",
     "is-eot": "1.0.0",
     "is-gif": "1.0.0",
-    "is-http2": "1.0.4",
+    "is-http2": "1.2.0",
     "is-jpg": "1.0.0",
     "is-otf": "0.1.2",
-    "is-png": "1.0.0",
-    "is-svg": "2.0.1",
+    "is-png": "1.1.0",
+    "is-svg": "2.1.0",
     "is-ttf": "0.2.2",
     "is-woff": "1.0.3",
     "is-woff2": "1.0.0",
-    "lwip": "0.0.9",
-    "md5": "2.1.0",
+    "jimp": "0.2.28",
+    "md5": "2.2.1",
     "meow": "3.7.0",
-    "minimize": "2.0.0",
+    "minimize": "2.1.0",
     "parse-color": "1.0.0",
-    "phantomas": "1.16.0",
-    "ps-node": "0.1.2",
-    "q": "1.4.1",
-    "request": "2.72.0",
-    "rimraf": "2.5.3",
+    "phantomas": "1.19.0",
+    "ps-node": "0.1.6",
+    "q": "1.5.1",
+    "request": "2.83.0",
+    "rimraf": "2.6.2",
     "temporary": "0.0.8",
-    "try-thread-sleep": "1.0.0",
-    "uglify-js": "2.7.0"
+    "try-thread-sleep": "1.0.2",
+    "uglify-js": "2.8.29"
   },
   "devDependencies": {
-    "chai": "~3.5.0",
-    "grunt": "~0.4.5",
-    "grunt-blanket": "~0.0.10",
-    "grunt-contrib-clean": "~1.0.0",
+    "chai": "~4.1.2",
+    "grunt": "~1.0.1",
+    "grunt-contrib-clean": "~1.1.0",
     "grunt-contrib-concat": "~1.0.1",
     "grunt-contrib-copy": "~1.0.0",
-    "grunt-contrib-cssmin": "~1.0.1",
-    "grunt-contrib-htmlmin": "~1.4.0",
-    "grunt-contrib-jshint": "~1.0.0",
-    "grunt-contrib-less": "~1.3.0",
-    "grunt-contrib-uglify": "~1.0.1",
+    "grunt-contrib-cssmin": "~2.2.1",
+    "grunt-contrib-htmlmin": "~2.4.0",
+    "grunt-contrib-jshint": "~1.1.0",
+    "grunt-contrib-less": "~1.4.1",
+    "grunt-contrib-uglify": "~3.3.0",
+    "grunt-contrib-watch": "~1.0.0",
     "grunt-env": "~0.4.4",
     "grunt-express": "~1.4.1",
     "grunt-filerev": "~2.3.1",
     "grunt-inline-angular-templates": "~0.1.5",
-    "grunt-line-remover": "~0.0.2",
-    "grunt-mocha-test": "~0.12.7",
+    "grunt-mocha-test": "~0.13.3",
+    "grunt-parallel": "~0.5.1",
     "grunt-replace": "~1.0.1",
     "grunt-usemin": "~3.1.1",
-    "grunt-webfont": "~1.4.0",
-    "matchdep": "~1.0.1",
-    "mocha": "~2.5.3",
-    "sinon": "~1.17.4",
-    "sinon-chai": "~2.8.0"
+    "grunt-webfont": "~1.6.0",
+    "matchdep": "~2.0.0",
+    "mocha": "~4.1.0",
+    "sinon": "~4.1.6",
+    "sinon-chai": "~2.14.0"
   },
   "scripts": {
     "test": "grunt test"

server_config/server_install.sh

@@ -3,10 +3,10 @@
 # APT-GET
 sudo apt-get update
 sudo apt-get install lsb-release libfontconfig1 libfreetype6 libjpeg-dev -y --force-yes > /dev/null 2>&1
-sudo apt-get install curl git python-software-properties build-essential make g++ -y --force-yes > /dev/null 2>&1
+sudo apt-get install curl git software-properties-common build-essential make g++ -y --force-yes > /dev/null 2>&1
 
 # Installation of NodeJS
-curl -sL https://deb.nodesource.com/setup_0.12 | sudo -E bash -
+curl -sL https://deb.nodesource.com/setup_4.x | sudo -E bash -
 sudo apt-get install -y nodejs > /dev/null 2>&1
 source ~/.profile
 
@@ -15,11 +15,8 @@ npm install forever grunt-cli -g
 source ~/.profile
 
 # Installation of YellowLabTools
-sudo mkdir /space
-sudo chown $USER /space
-cd /space
-git clone https://github.com/gmetais/YellowLabTools.git --branch master
-cd YellowLabTools
+sudo chown -R $USER /space
+cd /space/YellowLabTools
 npm install || exit 1
 
 # Front-end compilation

server_config/settings-prod.json

@@ -8,5 +8,6 @@
         
     },
     "maxAnonymousRunsPerDay": 99999999,
-    "maxAnonymousCallsPerDay": 99999999
+    "maxAnonymousCallsPerDay": 99999999,
+    "blockedUrls": []
 }

server_config/settings.json

@@ -8,5 +8,6 @@
         
     },
     "maxAnonymousRunsPerDay": 99999999,
-    "maxAnonymousCallsPerDay": 99999999
+    "maxAnonymousCallsPerDay": 99999999,
+    "blockedUrls": []
 }

test/api/apiTest.js

@@ -50,7 +50,8 @@ describe('api', function() {
             method: 'POST',
             url: serverUrl + '/api/runs',
             body: {
-                url: ''
+                url: '',
+                waitForResponse: true
             },
             json: true,
             headers: {
@@ -186,7 +187,7 @@ describe('api', function() {
                 body.should.not.have.a.property('screenshotBuffer');
                 // Check if the screenshot url is here
                 body.should.have.a.property('screenshotUrl');
-                body.screenshotUrl.should.equal('/api/results/' + body.runId + '/screenshot.jpg');
+                body.screenshotUrl.should.equal('api/results/' + body.runId + '/screenshot.jpg');
 
                 screenshotUrl = body.screenshotUrl;
 
@@ -659,7 +660,7 @@ describe('api', function() {
 
         request({
             method: 'GET',
-            url: serverUrl + screenshotUrl
+            url: serverUrl + '/' + screenshotUrl
         }, function(error, response, body) {
             if (!error && response.statusCode === 200) {
                 response.headers['content-type'].should.equal('image/jpeg');
@@ -686,4 +687,28 @@ describe('api', function() {
         });
     });
 
+    it('should refuse a query on a blocked Url', function(done) {
+        this.timeout(5000);
+
+        request({
+            method: 'POST',
+            url: serverUrl + '/api/runs',
+            body: {
+                url: 'http://www.test.com/something.html',
+                waitForResponse: false
+            },
+            json: true,
+            headers: {
+                'Content-Type': 'application/json',
+                'X-Api-Key': Object.keys(config.authorizedKeys)[0]
+            }
+        }, function(error, response, body) {
+            if (!error && response.statusCode === 403) {
+                done();
+            } else {
+                done(error || response.statusCode);
+            }
+        });
+    });
+
 });

test/api/screenshotHandlerTest.js

@@ -7,17 +7,17 @@ var path = require('path');
 describe('screenshotHandler', function() {
 
     var imagePath = path.join(__dirname, '../fixtures/logo-large.png');
-    var screenshot, lwipImage;
+    var screenshot, jimpImage;
 
-    
-    it('should open an image and return an lwip object', function(done) {
+
+    it('should open an image and return an jimp object', function(done) {
         ScreenshotHandler.openImage(imagePath)
             .then(function(image) {
-                lwipImage = image;
+                jimpImage = image;
 
-                lwipImage.should.be.an('object');
-                lwipImage.width().should.equal(620);
-                lwipImage.height().should.equal(104);
+                jimpImage.should.be.an('object');
+                jimpImage.bitmap.width.should.equal(620);
+                jimpImage.bitmap.height.should.equal(104);
 
                 done();
             })
@@ -26,14 +26,14 @@ describe('screenshotHandler', function() {
             });
     });
 
-    
-    it('should resize an lwip image', function(done) {
-        ScreenshotHandler.resizeImage(lwipImage, 310)
+
+    it('should resize an jimp image', function(done) {
+        ScreenshotHandler.resizeImage(jimpImage, 310)
             .then(function(image) {
-                lwipImage = image;
+                jimpImage = image;
 
-                lwipImage.width().should.equal(310);
-                lwipImage.height().should.equal(52);
+                jimpImage.bitmap.width.should.equal(310);
+                jimpImage.bitmap.height.should.equal(52);
 
                 done();
             })
@@ -43,8 +43,8 @@ describe('screenshotHandler', function() {
     });
 
 
-    it('should transform a lwip image into a buffer', function(done) {
-        ScreenshotHandler.toBuffer(lwipImage)
+    it('should transform a jimp image into a buffer', function(done) {
+        ScreenshotHandler.toBuffer(jimpImage)
             .then(function(buffer) {
                 buffer.should.be.an.instanceof(Buffer);
                 done();
@@ -125,4 +125,4 @@ describe('screenshotHandler', function() {
                 done(err);
             });
     });
-});
+});

test/blanket.js

@@ -1,7 +0,0 @@
-var path = require('path');
-var srcDir = path.join(__dirname, '..', 'server');
-
-require('blanket')({
-  // Only files that match the pattern will be instrumented
-  pattern: srcDir
-});

test/core/customPoliciesTest.js

@@ -182,22 +182,19 @@ describe('customPolicies', function() {
     it('should grade correctly jQuery versions', function() {
 
         var versions = {
-            '1.2.9': 0,
-            '1.4.4': 0,
-            '1.5.0': 0,
-            '1.6.3': 10,
-            '1.7.0': 20,
-            '1.8.3a': 30,
-            '1.9.2': 40,
-            '1.10.1': 50,
-            '2.0.0-rc1': 50,
-            '1.11.1': 70,
-            '2.1.1-beta1': 70,
-            '1.12.1': 90,
-            '2.2.1': 90,
-            '3.0.1': 100,
-            '3.1.0': 100,
-            '3.2.1': 100
+            '1.7.0': 0,
+            '1.10.1': 20,
+            '1.10.3a': 20,
+            '2.0.0-rc1': 20,
+            '1.11.1': 30,
+            '2.1.1-beta1': 30,
+            '1.12.1': 40,
+            '2.2.1': 40,
+            '3.0.1': 50,
+            '3.1.0': 70,
+            '3.2.1': 90,
+            '3.3.1': 100,
+            '3.5.0': 100
         };
 
         for (var version in versions) {

test/core/fileMinifierTest.js

@@ -37,7 +37,7 @@ describe('fileMinifier', function() {
             type: 'js',
             contentLength: 999,
             weightCheck: {
-                body: fileContent.toString('utf8'),
+                bodyBuffer: fileContent,
                 totalWeight: fileSize + 200,
                 headersSize: 200,
                 bodySize: fileSize,
@@ -179,7 +179,7 @@ describe('fileMinifier', function() {
             type: 'css',
             contentLength: 999,
             weightCheck: {
-                body: fileContent.toString('utf8'),
+                bodyBuffer: fileContent,
                 totalWeight: fileSize + 200,
                 headersSize: 200,
                 bodySize: fileSize,

test/core/fontAnalyzerTest.js

@@ -0,0 +1,114 @@
+var should = require('chai').should();
+var fontAnalyzer = require('../../lib/tools/redownload/fontAnalyzer');
+var fs = require('fs');
+var path = require('path');
+
+describe('fontAnalyzer', function() {
+    
+    it('should extract metrics from a font', function(done) {
+        this.timeout(10000);
+
+        var fileContent = fs.readFileSync(path.resolve(__dirname, '../www/SourceSansPro/SourceSansPro-Regular.woff'));
+        var fileSize = fileContent.length;
+
+        var entry = {
+            method: 'GET',
+            url: 'http://localhost:8388/SourceSansPro/SourceSansPro-Regular.woff',
+            requestHeaders: {
+                'User-Agent': 'something',
+                Referer: 'http://www.google.fr/',
+                Accept: '*/*',
+                'Accept-Encoding': 'gzip, deflate'
+            },
+            status: 200,
+            isWebFont: true,
+            type: 'webfont',
+            contentType: 'image/jpeg',
+            contentLength: 999,
+            weightCheck: {
+                bodyBuffer: fileContent,
+                totalWeight: fileSize + 200,
+                headersSize: 200,
+                bodySize: fileSize,
+                isCompressed: false,
+                uncompressedSize: fileSize
+            }
+        };
+
+        fontAnalyzer.getMetricsFromFont(entry, 'ABCD')
+
+        .then(function(metrics) {
+
+            metrics.should.be.an('Object');
+            metrics.should.have.a.property('name').that.equals('Source Sans Pro');
+            metrics.should.have.a.property('numGlyphs').that.equals(1944);
+            metrics.should.have.a.property('averageGlyphComplexity').that.equals(26.6);
+            metrics.should.have.a.property('compressedWeight').that.equals(fileSize);
+
+            metrics.should.have.a.property('unicodeRanges').that.is.an('Object');
+            metrics.unicodeRanges.should.have.a.property('Basic Latin');
+            metrics.unicodeRanges['Basic Latin'].should.have.a.property('charset').that.is.a('String');
+            metrics.unicodeRanges['Basic Latin'].charset.length.should.equal(95);
+            metrics.unicodeRanges['Basic Latin'].name.should.equal('Basic Latin');
+            metrics.unicodeRanges['Basic Latin'].rangeStart.should.equal(0x0020);
+            metrics.unicodeRanges['Basic Latin'].rangeEnd.should.equal(0x007F);
+            metrics.unicodeRanges['Basic Latin'].coverage.should.equal(95 / 96);
+            metrics.unicodeRanges['Basic Latin'].numGlyphsInCommonWithPageContent.should.equal(4);
+
+            metrics.unicodeRanges.Cyrillic.numGlyphsInCommonWithPageContent.should.equal(0);
+
+            metrics.should.have.a.property('numGlyphsInCommonWithPageContent').that.equals(4);
+
+            should.equal(metrics.unicodeRanges.Others.coverage, undefined);
+
+            done();
+        })
+
+        .fail(function(err) {
+            done(err);
+        });
+    });
+
+    it('should sort glyphes by unicode ranges', function() {
+        var ranges = fontAnalyzer.readUnicodeRanges([0x0041, 0x0042, 0x0043, 0x0044, 0x0416], '0123AMZ');
+        
+        ranges.should.deep.equal({ 
+            'Basic Latin': {
+                name: 'Basic Latin',
+                rangeStart: 32,
+                rangeEnd: 127,
+                charset: 'ABCD',
+                coverage: 0.041666666666666664,
+                numGlyphsInCommonWithPageContent: 1
+            },
+            'Cyrillic': {
+                name: 'Cyrillic',
+                rangeStart: 1024,
+                rangeEnd: 1327,
+                charset: 'Ж',
+                coverage: 0.003289473684210526,
+                numGlyphsInCommonWithPageContent: 0
+            }
+        });
+    });
+
+    it('should transform an array of char codes into a string', function() {
+        var str = fontAnalyzer.getCharacterSetAsString([0x0041, 0x0042, 0x0043, 0x0044, 0x0416]);
+        str.should.equal('ABCDЖ');
+    });
+
+    it('should find the right unicode range for a char', function() {
+        fontAnalyzer.getUnicodeRangeFromChar(0x0020).should.deep.equal({
+            name: 'Basic Latin',
+            rangeStart: 0x0020,
+            rangeEnd: 0x007F
+        });
+
+        fontAnalyzer.getUnicodeRangeFromChar(0x0021).name.should.equal('Basic Latin');
+        fontAnalyzer.getUnicodeRangeFromChar(0x007F).name.should.equal('Basic Latin');
+        fontAnalyzer.getUnicodeRangeFromChar(0x007E).name.should.equal('Basic Latin');
+        fontAnalyzer.getUnicodeRangeFromChar(0x0000).name.should.equal('Others');
+        fontAnalyzer.getUnicodeRangeFromChar(0xFFFFFFFFF).name.should.equal('Others');
+    });
+
+});

test/core/gzipCompressorTest.js

@@ -23,7 +23,7 @@ describe('gzipCompressor', function() {
             type: 'js',
             contentLength: 999,
             weightCheck: {
-                body: fileContent.toString('utf8'),
+                bodyBuffer: fileContent,
                 totalWeight: fileSize + 200,
                 headersSize: 200,
                 bodySize: fileSize,
@@ -64,7 +64,7 @@ describe('gzipCompressor', function() {
             type: 'js',
             contentLength: 999,
             weightCheck: {
-                body: fileContent.toString('utf8'),
+                bodyBuffer: fileContent,
                 bodyAfterOptimization: minifiedContent.toString('utf8'),
                 totalWeight: fileSize + 200,
                 headersSize: 200,
@@ -108,7 +108,7 @@ describe('gzipCompressor', function() {
             type: 'js',
             contentLength: 999,
             weightCheck: {
-                body: fileContent.toString('utf8'),
+                bodyBuffer: fileContent,
                 bodyAfterOptimization: minifiedContent.toString('utf8'),
                 totalWeight: gzipedSize + 200,
                 headersSize: 200,
@@ -150,7 +150,7 @@ describe('gzipCompressor', function() {
             type: 'js',
             contentLength: 999,
             weightCheck: {
-                body: fileContent.toString('utf8'),
+                bodyBuffer: fileContent,
                 totalWeight: gzipedSize + 200,
                 headersSize: 200,
                 bodySize: gzipedSize,
@@ -188,7 +188,7 @@ describe('gzipCompressor', function() {
             type: 'css',
             contentLength: 999,
             weightCheck: {
-                body: fileContent.toString('utf8'),
+                bodyBuffer: fileContent,
                 totalWeight: fileSize + 200,
                 headersSize: 200,
                 bodySize: fileSize,
@@ -224,7 +224,7 @@ describe('gzipCompressor', function() {
             type: 'html',
             contentLength: 999,
             weightCheck: {
-                body: fileContent.toString('utf8'),
+                bodyBuffer: fileContent,
                 totalWeight: fileSize + 200,
                 headersSize: 200,
                 bodySize: fileSize,
@@ -261,7 +261,7 @@ describe('gzipCompressor', function() {
             type: 'image',
             contentLength: 999,
             weightCheck: {
-                body: fileContent.toString('utf8'),
+                bodyBuffer: fileContent,
                 totalWeight: fileSize + 200,
                 headersSize: 200,
                 bodySize: fileSize,
@@ -297,7 +297,7 @@ describe('gzipCompressor', function() {
             type: 'xml',
             contentLength: 999,
             weightCheck: {
-                body: fileContent.toString('utf8'),
+                bodyBuffer: fileContent,
                 totalWeight: fileSize + 200,
                 headersSize: 200,
                 bodySize: fileSize,
@@ -333,7 +333,7 @@ describe('gzipCompressor', function() {
             type: 'json',
             contentLength: 999,
             weightCheck: {
-                body: fileContent.toString('utf8'),
+                bodyBuffer: fileContent,
                 totalWeight: fileSize + 200,
                 headersSize: 200,
                 bodySize: fileSize,
@@ -370,7 +370,7 @@ describe('gzipCompressor', function() {
             type: 'webfont',
             contentLength: 999,
             weightCheck: {
-                body: fileContent.toString('utf8'),
+                bodyBuffer: fileContent,
                 totalWeight: fileSize + 200,
                 headersSize: 200,
                 bodySize: fileSize,
@@ -407,7 +407,7 @@ describe('gzipCompressor', function() {
             type: 'favicon',
             contentLength: 999,
             weightCheck: {
-                body: fileContent.toString('utf8'),
+                bodyBuffer: fileContent,
                 totalWeight: fileSize + 200,
                 headersSize: 200,
                 bodySize: fileSize,
@@ -444,7 +444,7 @@ describe('gzipCompressor', function() {
             contentType: 'image/jpeg',
             contentLength: 999,
             weightCheck: {
-                body: fileContent.toString('utf8'),
+                bodyBuffer: fileContent,
                 totalWeight: fileSize + 200,
                 headersSize: 200,
                 bodySize: fileSize,
@@ -482,7 +482,7 @@ describe('gzipCompressor', function() {
             contentType: 'image/png',
             contentLength: 999,
             weightCheck: {
-                body: fileContent.toString('utf8'),
+                bodyBuffer: fileContent,
                 totalWeight: fileSize + 200,
                 headersSize: 200,
                 bodySize: fileSize,
@@ -519,7 +519,7 @@ describe('gzipCompressor', function() {
             contentType: 'image/gif',
             contentLength: 999,
             weightCheck: {
-                body: fileContent.toString('utf8'),
+                bodyBuffer: fileContent,
                 totalWeight: fileSize + 200,
                 headersSize: 200,
                 bodySize: fileSize,
@@ -556,7 +556,7 @@ describe('gzipCompressor', function() {
             contentType: 'image/webp',
             contentLength: 999,
             weightCheck: {
-                body: fileContent.toString('utf8'),
+                bodyBuffer: fileContent,
                 totalWeight: fileSize + 200,
                 headersSize: 200,
                 bodySize: fileSize,

test/core/imageOptimizerTest.js

@@ -52,7 +52,7 @@ describe('imageOptimizer', function() {
             contentType: 'image/jpeg',
             contentLength: 999,
             weightCheck: {
-                body: fileContent,
+                bodyBuffer: fileContent,
                 totalWeight: fileSize + 200,
                 headersSize: 200,
                 bodySize: fileSize,
@@ -137,7 +137,7 @@ describe('imageOptimizer', function() {
             contentType: 'image/png',
             contentLength: 999,
             weightCheck: {
-                body: fileContent,
+                bodyBuffer: fileContent,
                 totalWeight: fileSize + 200,
                 headersSize: 200,
                 bodySize: fileSize,
@@ -194,7 +194,7 @@ describe('imageOptimizer', function() {
             contentType: 'image/svg+xml',
             contentLength: 999,
             weightCheck: {
-                body: fileContent,
+                bodyBuffer: fileContent,
                 totalWeight: fileSize + 200,
                 headersSize: 200,
                 bodySize: fileSize,
@@ -242,7 +242,7 @@ describe('imageOptimizer', function() {
             contentType: 'image/jpeg',
             contentLength: 999,
             weightCheck: {
-                body: fileContent,
+                bodyBuffer: fileContent,
                 totalWeight: fileSize + 200,
                 headersSize: 200,
                 bodySize: fileSize,
@@ -285,7 +285,7 @@ describe('imageOptimizer', function() {
             contentType: 'image/png',
             contentLength: 999,
             weightCheck: {
-                body: fileContent,
+                bodyBuffer: fileContent,
                 totalWeight: fileSize + 200,
                 headersSize: 200,
                 bodySize: fileSize,

test/core/indexTest.js

@@ -66,9 +66,9 @@ describe('index.js', function() {
                         "tool": "phantomas",
                         "label": "DOM max depth",
                         "message": "<p>A deep DOM makes the CSS matching with DOM elements difficult.</p><p>It also slows down JavaScript modifications to the DOM because changing the dimensions of an element makes the browser re-calculate the dimensions of it's parents. Same thing for JavaScript events, that bubble up to the document root.</p>",
-                        "isOkThreshold": 10,
-                        "isBadThreshold": 20,
-                        "isAbnormalThreshold": 28,
+                        "isOkThreshold": 12,
+                        "isBadThreshold": 22,
+                        "isAbnormalThreshold": 30,
                         "hasOffenders": true
                     },
                     "value": 1,
@@ -109,6 +109,21 @@ describe('index.js', function() {
             });
     });
 
+    it('should succeed on try-catch.html', function(done) {
+        this.timeout(15000);
+
+        var url = 'http://localhost:8388/try-catch.html';
+
+        ylt(url)
+            .then(function(data) {
+                data.toolsResults.phantomas.metrics.should.have.a.property('jsErrors').that.equals(0);
+                done();
+            }).fail(function(err) {
+                console.log.restore();
+                done(err);
+            });
+    });
+
     it('should take a screenshot', function(done) {
         this.timeout(15000);
 

test/core/redownloadTest.js

@@ -195,7 +195,7 @@ describe('redownload', function() {
             newEntry.weightCheck.uncompressedSize.should.equal(newEntry.weightCheck.bodySize);
             newEntry.weightCheck.isCompressed.should.equal(false);
             newEntry.weightCheck.headersSize.should.be.above(200).and.below(400);
-            newEntry.weightCheck.body.toString().should.have.string('1.8.3');
+            newEntry.weightCheck.bodyBuffer.toString().should.have.string('1.8.3');
 
             done();
         })
@@ -226,17 +226,16 @@ describe('redownload', function() {
         .then(function(newEntry) {
 
             newEntry.weightCheck.bodySize.should.equal(4193);
-            newEntry.weightCheck.body.should.equal(fileContent.toString('binary'));
+            newEntry.weightCheck.bodyBuffer.should.deep.equal(fileContent);
 
-            // Opening the image in lwip to check if the format is good
-            var lwip = require('lwip');
-            var buffer = new Buffer(newEntry.weightCheck.body, 'binary');
-            lwip.open(buffer, 'png', function(err, image) {
-                image.width().should.equal(620);
-                image.height().should.equal(104);
+            // Opening the image in jimp to check if the format is good
+            var Jimp = require('jimp');
+            Jimp.read(newEntry.weightCheck.bodyBuffer, function(err, image) {
+                image.bitmap.width.should.equal(620);
+                image.bitmap.height.should.equal(104);
                 done(err);
             });
-            
+
         })
 
         .fail(function(err) {
@@ -312,7 +311,7 @@ describe('redownload', function() {
             type: 'html',
             contentLength: 999,
             weightCheck: {
-                body: 'blabla',
+                bodyBuffer: 'blabla',
                 headersSize: 200,
                 bodySize: 500,
                 isCompressed: false,

test/fixtures/settings.json

@@ -8,5 +8,8 @@
         "1234567890": "contact@gaelmetais.com"
     },
     "maxAnonymousRunsPerDay": 10,
-    "maxAnonymousCallsPerDay": 1000
+    "maxAnonymousCallsPerDay": 1000,
+    "blockedUrls": [
+        "http://www.test.com"
+    ]
 }

test/www/SourceSansPro/OFL.txt

@@ -0,0 +1,93 @@
+Copyright 2010, 2012 Adobe Systems Incorporated (http://www.adobe.com/), with Reserved Font Name 'Source'. All Rights Reserved. Source is a trademark of Adobe Systems Incorporated in the United States and/or other countries.
+
+This Font Software is licensed under the SIL Open Font License, Version 1.1.
+
+This license is copied below, and is also available with a FAQ at: http://scripts.sil.org/OFL
+
+
+-----------------------------------------------------------
+SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007
+-----------------------------------------------------------
+
+PREAMBLE
+The goals of the Open Font License (OFL) are to stimulate worldwide
+development of collaborative font projects, to support the font creation
+efforts of academic and linguistic communities, and to provide a free and
+open framework in which fonts may be shared and improved in partnership
+with others.
+
+The OFL allows the licensed fonts to be used, studied, modified and
+redistributed freely as long as they are not sold by themselves. The
+fonts, including any derivative works, can be bundled, embedded, 
+redistributed and/or sold with any software provided that any reserved
+names are not used by derivative works. The fonts and derivatives,
+however, cannot be released under any other type of license. The
+requirement for fonts to remain under this license does not apply
+to any document created using the fonts or their derivatives.
+
+DEFINITIONS
+"Font Software" refers to the set of files released by the Copyright
+Holder(s) under this license and clearly marked as such. This may
+include source files, build scripts and documentation.
+
+"Reserved Font Name" refers to any names specified as such after the
+copyright statement(s).
+
+"Original Version" refers to the collection of Font Software components as
+distributed by the Copyright Holder(s).
+
+"Modified Version" refers to any derivative made by adding to, deleting,
+or substituting -- in part or in whole -- any of the components of the
+Original Version, by changing formats or by porting the Font Software to a
+new environment.
+
+"Author" refers to any designer, engineer, programmer, technical
+writer or other person who contributed to the Font Software.
+
+PERMISSION & CONDITIONS
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of the Font Software, to use, study, copy, merge, embed, modify,
+redistribute, and sell modified and unmodified copies of the Font
+Software, subject to the following conditions:
+
+1) Neither the Font Software nor any of its individual components,
+in Original or Modified Versions, may be sold by itself.
+
+2) Original or Modified Versions of the Font Software may be bundled,
+redistributed and/or sold with any software, provided that each copy
+contains the above copyright notice and this license. These can be
+included either as stand-alone text files, human-readable headers or
+in the appropriate machine-readable metadata fields within text or
+binary files as long as those fields can be easily viewed by the user.
+
+3) No Modified Version of the Font Software may use the Reserved Font
+Name(s) unless explicit written permission is granted by the corresponding
+Copyright Holder. This restriction only applies to the primary font name as
+presented to the users.
+
+4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font
+Software shall not be used to promote, endorse or advertise any
+Modified Version, except to acknowledge the contribution(s) of the
+Copyright Holder(s) and the Author(s) or with their explicit written
+permission.
+
+5) The Font Software, modified or unmodified, in part or in whole,
+must be distributed entirely under this license, and must not be
+distributed under any other license. The requirement for fonts to
+remain under this license does not apply to any document created
+using the Font Software.
+
+TERMINATION
+This license becomes null and void if any of the above conditions are
+not met.
+
+DISCLAIMER
+THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
+OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE
+COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL
+DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM
+OTHER DEALINGS IN THE FONT SOFTWARE.

test/www/font-page.html

@@ -0,0 +1,23 @@
+<html>
+<head>
+<style>
+    body {
+        font-size: 4em;
+    }
+    script, style {
+        display: block;
+        background: #F66;
+    }
+    @font-face {
+        font-family: "SourceSansPro";
+        src: url("SourceSansPro/SourceSansPro-Regular.woff");
+    }
+    .foo {
+        font-family: SourceSansPro;
+    }
+</style>
+</head>
+<body>
+<div class="foo">Some text</div>
+</body>
+</html>

test/www/try-catch.html

@@ -0,0 +1,21 @@
+<html>
+    <head>
+        <title>Testing getElementById with a try catch statement</title>
+    </head>
+    <body>
+        <div id="foo">Some text</div>
+        <script>
+            try {
+                document.getElementById(undefined);
+                document.getElementsByClassName(undefined);
+                document.getElementsByTagName(undefined);
+                
+            } catch(err) {
+                console.log('Error found: ' + err);
+                throw new Error("I detect an error!");
+            }
+
+            document.getElementById('foo');
+        </script>
+    </body>
+</html>