Merge pull request #338 from YellowLabTools/develop

Merge develop into master

front/src/views/rule.html

@@ -179,10 +179,18 @@
                         (<ng-pluralize count="offender.requests" when="{'one':'1 request','other':'{} requests'}"></ng-pluralize>)
                     </div>
 
-                    <div ng-if="policyName === 'globalVariables' || policyName === 'jQueryVersionsLoaded' || policyName === 'synchronousXHR'">
+                    <div ng-if="policyName === 'globalVariables'">
+                        {{offender}}
+                    </div>
+
+                    <div ng-if="policyName === 'jQueryVersionsLoaded'">
                         {{offender.version}}
                     </div>
 
+                    <div ng-if="policyName === 'synchronousXHR'">
+                        {{offender.url}}
+                    </div>
+
                     <div ng-if="policyName === 'fontsCount'">
                         <url-link url="offender.url" max-length="70"></url-link>
                         ({{offender.size | bytes}})

lib/metadata/policies.js

@@ -538,7 +538,7 @@ var policies = {
         "message": "<p>It can be useful, but only as a last resort. It is a bad practice because it overrides the normal cascading logic. The more you use !important, the more you need it again to over-override. This conducts to a poor maintainability.</p>",
         "isOkThreshold": 0,
         "isBadThreshold": 200,
-        "isAbnormalThreshold": 500,
+        "isAbnormalThreshold": 1000,
         "hasOffenders": true,
         "offendersTransformFn": function(offenders) {
             var parsedOffenders = offenders.map(function(offender) {
@@ -757,8 +757,8 @@ var policies = {
     },
     "compression": {
         "tool": "redownload",
-        "label": "Brotli compression",
-        "message": "<p>Measures the number of bytes that could be saved by compressing textual files. Some files listed below might not be compressed at all, some might be already compressed with Gzip but would become even lighter with Brotli.</p><p>All major server systems are now compatible with Brotli.</p><p>Note that compressing small files (< 1 KB) is arguable, and that some assets such as images should not be compressed as it is already included in their format. <a href=\"https://gist.github.com/gmetais/971ce13a1fbeebd88445\" target=\"_blank\">Here</a> is a list of Content-Types that should be compressed.</p>",
+        "label": "Gzip/Brotli compression",
+        "message": "<p>Measures the number of bytes that could be saved by compressing textual files. Some files listed below might not be compressed at all, some might be already compressed with Gzip but would become even lighter with Brotli.</p><p>All major server systems are now compatible with Brotli.</p><p>Note that compressing small files (< 1 KB) is arguable, and that some assets such as images should not be compressed as it is already included in their format. <a href=\"https://letstalkaboutwebperf.com/en/gzip-brotli-server-config/\" target=\"_blank\">Here</a> is a list of MIME types that should be compressed.</p>",
         "isOkThreshold": 20480,
         "isBadThreshold": 204800,
         "isAbnormalThreshold": 409600,
@@ -875,7 +875,7 @@ var policies = {
             return offenders;
         }
     },
-    /*"unusedUnicodeRanges": {
+    "unusedUnicodeRanges": {
         "tool": "redownload",
         "label": "Unused Unicode ranges",
         "message": "<p>This metric counts the number of unused Unicode ranges inside each font. For example, one font could include Cyrillic glyphs but none of them are used on the page.</p><p>It also reveals the number of ligatures (letters that are represented differently when close to each other) and hidden chars (glyphs not linked to the unicode system that can't be displayed on the web).</p><p>Because of technical limitations, Yellow Lab Tools checks each font against the glyphs of the entire page. As a result, estimated use is >= to reality. For example, if you read that 10 glyphs are \"possibly used\", it means that these 10 glyphs are used on the page but nothing guaranties that they are displayed using this font.</p><p>Tools such as <a href=\"https://www.fontsquirrel.com/tools/webfont-generator\" target=\"_blank\">Font Squirrel</a> can remove some unicode ranges from a font.</p><p>In the case of an icon font, make sure you only keep the icons that are used on the website and to remove the others. Several tools are able to extract SVG images from a font, then some other tools can generate a font from the SVGs you want to keep.</p>",
@@ -886,7 +886,7 @@ var policies = {
         "offendersTransformFn": function(offenders) {
             return offenders;
         }
-    },*/
+    },
     "nonWoff2Fonts": {
         "tool": "redownload",
         "label": "WOFF 2",

lib/metadata/scoreProfileGeneric.json

@@ -86,6 +86,7 @@
             "policies": {
                 "fontsCount": 1,
                 "heavyFonts": 0.5,
+                "unusedUnicodeRanges": 0.5,
                 "nonWoff2Fonts": 0.5
             }
         },

lib/offendersHelpers.js

@@ -117,7 +117,7 @@ var OffendersHelpers = function() {
 
             for (var i=0 ; i<traceArray.length ; i++) {
                 // Handle the new PhantomJS 2.x syntax
-                parts = /^(([\w$]+)@)?([^ ]+):(\d+):(\d+)$/.exec(traceArray[i]);
+                parts = /^\s*at( (\w+))? \(?([^ ]+):(\d+):(\d+)\)?$$/.exec(traceArray[i]);
 
                 if (parts) {
                     obj = {
@@ -132,25 +132,6 @@ var OffendersHelpers = function() {
 
                     results.push(obj);
 
-                } else {
-                    // Old syntax
-                    parts = /^(([\w$]+) )?\(?([^ ]+):(\d+)\)?$/.exec(traceArray[i]);
-
-                    if (parts) {
-                        obj = {
-                            file: parts[3],
-                            line: parseInt(parts[4], 10)
-                        };
-
-                        if (parts[2]) {
-                            obj.functionName = parts[2];
-                        }
-
-                        results.push(obj);
-
-                    } else {
-                        return null;
-                    }
                 }
             }
             return results;

lib/runner.js

@@ -78,6 +78,12 @@ var Runner = function(params) {
             milestone: 'redownload'
         });
 
+        // Fix: don't display Unicode ranges if the module is not present in Phantomas
+        if (!data.toolsResults.phantomas.metrics.charactersCount) {
+            delete data.toolsResults.redownload.metrics.unusedUnicodeRanges;
+            delete data.toolsResults.redownload.offenders.unusedUnicodeRanges;
+        }
+
         // Rules checker
         var policies = require('./metadata/policies');
         data.rules = rulesChecker.check(data, policies);

lib/server/controllers/apiController.js

@@ -227,7 +227,10 @@ var ApiController = function(app) {
     // Counts all pending runs
     app.get('/api/runs', function(req, res) {
         res.setHeader('Content-Type', 'application/json');
-        res.send(JSON.stringify({pendingRuns: queue.length()}, null, 2));
+        res.send(JSON.stringify({
+            pendingRuns: queue.length(),
+            timeSinceLastTestStarted: queue.timeSinceLastTestStarted()
+        }, null, 2));
     });
 
     // Delete one run by id

lib/server/datastores/runsQueue.js

@@ -6,7 +6,7 @@ function RunsQueue() {
     'use strict';
 
     var queue = [];
-
+    var lastTestTimestamp = 0;
 
     this.push = function(runId) {
         var deferred = Q.defer();
@@ -21,6 +21,7 @@ function RunsQueue() {
                 runId: runId
             });
             
+            lastTestTimestamp = Date.now();
             deferred.resolve();
 
         } else {
@@ -31,6 +32,7 @@ function RunsQueue() {
                     deferred.notify(position);
                 },
                 itIsTimeCallback: function() {
+                    lastTestTimestamp = Date.now();
                     deferred.resolve();
                 }
             });
@@ -78,6 +80,11 @@ function RunsQueue() {
     this.length = function() {
         return queue.length;
     };
+
+    // Returns the number of seconds since the last test was launched
+    this.timeSinceLastTestStarted = function() {
+        return Math.round((Date.now() - lastTestTimestamp) / 1000);
+    };
 }
 
 module.exports = RunsQueue;

lib/server/middlewares/apiLimitsMiddleware.js

@@ -13,7 +13,14 @@ var apiLimitsMiddleware = function(req, res, next) {
     if (req.path.indexOf('/api/') === 0 && !res.locals.hasApiKey) {
         
         
-        if (req.path === '/api/runs') {
+        // Monitoring requests
+        if (req.path === '/api/runs' && req.method === 'GET') {
+            next();
+            return;
+        }
+
+        // New tests 
+        if (req.path === '/api/runs' && req.method === 'POST') {
             
             if (!runsTable.accepts(ipAddress)) {
                 // Sorry :/
@@ -24,6 +31,7 @@ var apiLimitsMiddleware = function(req, res, next) {
 
         }
 
+        // Every other calls
         if (!callsTable.accepts(ipAddress)) {
             // Sorry :/
             debug('Too many API requests from IP address %s', ipAddress);

lib/tools/phantomas/phantomasWrapper.js

@@ -47,7 +47,8 @@ var PhantomasWrapper = function() {
 
             // Mandatory
             'analyze-css': true,
-            'ignore-ssl-errors': true
+            'ignoreSslErrors': true, // until Phantomas 2.1
+            'ignore-ssl-errors': true // for Phantomas >= 2.2
         };
 
 

lib/tools/redownload/contentTypeChecker.js

@@ -10,6 +10,7 @@ var isWoff2 = require('is-woff2');
 var isOtf   = require('is-otf');
 var isTtf   = require('is-ttf');
 var isEot   = require('is-eot');
+var isJson  = require('is-json');
 
 var ContentTypeChecker = function() {
 
@@ -57,7 +58,7 @@ var ContentTypeChecker = function() {
                 foundType = findContentType(entry.weightCheck.bodyBuffer);
 
                 // If it's an image or a font, then rewrite.
-                if (foundType !== null && (foundType.type === 'image' || foundType.type === 'webfont')) {
+                if (foundType !== null && (foundType.type === 'image' || foundType.type === 'webfont' || foundType.type === 'json')) {
                     if (foundType.type !== entry.type) {
                         debug('Content type %s is wrong for %s. It should be %s.', entry.type, entry.ulr, foundType.type);
                     }
@@ -119,6 +120,10 @@ var ContentTypeChecker = function() {
             return contentTypes.eot;
         }
 
+        if (isJson(bodyStr)) {
+            return contentTypes.json;
+        }
+
         return null;
     }
 
@@ -224,7 +229,16 @@ var ContentTypeChecker = function() {
                 entry.type = 'webfont';
                 entry.isWebFont = true;
             }
-        }
+        },
+        json: {
+            type: 'json',
+            mimes: ['application/json'],
+            updateFn: function(entry) {
+                entry.type = 'json';
+                entry.isJSON = true;
+            }
+        },
+        
     };
     
     return {

lib/tools/redownload/fileMinifier.js

@@ -224,7 +224,7 @@ var FileMinifier = function() {
         return result;
     }
 
-    // Avoid losing some trying to compress JS files if they already look minified
+    // Avoid losing time trying to compress JS files if they already look minified
     // by counting the number of lines compared to the total size.
     // Less than 2KB per line is suspicious
     function looksAlreadyMinified(code) {

lib/tools/redownload/imageOptimizer.js

@@ -210,7 +210,7 @@ var ImageOptimizer = function() {
         }
 
 
-        imagemin.buffer(imageBody, {use: engine})
+        imagemin.buffer(imageBody, {plugins: [engine]})
 
         .then(function(file) {
             var endTime = Date.now();

lib/tools/redownload/redownload.js

@@ -65,8 +65,8 @@ var Redownload = function() {
 
         // Prevent a bug with the font analyzer on empty pages
         var differentCharacters = '';
-        if (data.toolsResults.phantomas.offenders.differentCharacters && data.toolsResults.phantomas.offenders.differentCharacters.length > 0) {
-            differentCharacters = data.toolsResults.phantomas.offenders.differentCharacters[0];
+        if (data.toolsResults.phantomas.offenders.charactersCount && data.toolsResults.phantomas.offenders.charactersCount.length > 0) {
+            differentCharacters = data.toolsResults.phantomas.offenders.charactersCount[0];
         }
 
         // Transform every request into a download function with a callback when done
@@ -823,6 +823,7 @@ var Redownload = function() {
         reqHeaders['Accept'] = '*/*,image/webp';
         reqHeaders['Accept-Encoding'] = 'gzip, deflate, br';
         reqHeaders['Connection'] = 'keep-alive';
+        reqHeaders['User-Agent'] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.182 Safari/537.36';
 
         var requestOptions = {
             method: entry.method,
@@ -899,6 +900,7 @@ var Redownload = function() {
                     var result = {
                         bodyBuffer: body,
                         headersSize: Buffer.byteLength(rawHeaders, 'utf8'),
+                        headers: res.headers,
                         bodySize: bodySize,
                         isCompressed: isCompressed,
                         compressionTool: compressionTool,

package.json

@@ -1,6 +1,6 @@
 {
   "name": "yellowlabtools",
-  "version": "2.0.0-beta",
+  "version": "2.0.0",
   "description": "Online tool to audit a webpage for performance and front-end quality issues",
   "license": "GPL-2.0",
   "author": {
@@ -42,14 +42,15 @@
     "fontkit": "1.7.8",
     "html-minifier": "4.0.0",
     "image-size": "0.7.1",
-    "imagemin": "6.1.0",
-    "imagemin-jpegoptim": "6.0.0",
-    "imagemin-jpegtran": "6.0.0",
-    "imagemin-optipng": "6.0.0",
-    "imagemin-svgo": "7.0.0",
+    "imagemin": "7.0.1",
+    "imagemin-jpegoptim": "7.0.0",
+    "imagemin-jpegtran": "7.0.0",
+    "imagemin-optipng": "8.0.0",
+    "imagemin-svgo": "8.0.0",
     "is-eot": "1.0.0",
     "is-gif": "3.0.0",
     "is-jpg": "2.0.0",
+    "is-json": "2.0.1",
     "is-otf": "0.1.2",
     "is-png": "1.1.0",
     "is-svg": "3.0.0",
@@ -67,7 +68,7 @@
     "request": "2.88.0",
     "rimraf": "2.6.3",
     "temporary": "0.0.8",
-    "ttf2woff2": "3.0.0",
+    "ttf2woff2": "4.0.1",
     "uglify-js": "3.4.9",
     "woff-tools": "0.1.0"
   },