From eedb46e000828f58671e0812989c7ad9105c01db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABl=20M=C3=A9tais?= Date: Mon, 11 Jul 2016 15:46:00 +0800 Subject: [PATCH] New rule called "Empty requests" (#180) --- front/src/views/rule.html | 3 +- lib/metadata/policies.js | 21 +++-- lib/metadata/scoreProfileGeneric.json | 1 + lib/runner.js | 4 +- .../contentTypeChecker.js | 0 .../fileMinifier.js | 0 .../gzipCompressor.js | 0 .../imageOptimizer.js | 0 .../redownload.js} | 38 ++++++-- test/core/contentTypeCheckerTest.js | 2 +- test/core/fileMinifierTest.js | 2 +- test/core/gzipCompressorTest.js | 4 +- test/core/imageOptimizerTest.js | 2 +- ...weightCheckerTest.js => redownloadTest.js} | 86 +++++++++---------- 14 files changed, 99 insertions(+), 64 deletions(-) rename lib/tools/{weightChecker => redownload}/contentTypeChecker.js (100%) rename lib/tools/{weightChecker => redownload}/fileMinifier.js (100%) rename lib/tools/{weightChecker => redownload}/gzipCompressor.js (100%) rename lib/tools/{weightChecker => redownload}/imageOptimizer.js (100%) rename lib/tools/{weightChecker/weightChecker.js => redownload/redownload.js} (94%) rename test/core/{weightCheckerTest.js => redownloadTest.js} (70%) diff --git a/front/src/views/rule.html b/front/src/views/rule.html index ee8fa78..769f378 100644 --- a/front/src/views/rule.html +++ b/front/src/views/rule.html @@ -141,7 +141,7 @@ -
+
@@ -158,7 +158,6 @@
{{offender}}
-
diff --git a/lib/metadata/policies.js b/lib/metadata/policies.js index 869f50a..0f52db5 100644 --- a/lib/metadata/policies.js +++ b/lib/metadata/policies.js @@ -899,7 +899,7 @@ var policies = { } }, "totalWeight": { - "tool": "weightChecker", + "tool": "redownload", "label": "Total weight", "message": "

The weight is of course very important if you want the page to load fast. Try to stay under 1MB, which is alreay very long to download over a slow connection.

Please note that Yellow Lab Tools' engine (PhantomJS) is not compatible with image srcset (unless you use a polyfill). This can lead to incorrect page weight.

", "isOkThreshold": 716800, @@ -909,7 +909,7 @@ var policies = { "unit": 'bytes' }, "imageOptimization": { - "tool": "weightChecker", + "tool": "redownload", "label": "Image optimization", "message": "

This metric measures the number of bytes that could be saved by optimizing images.

Image optimization is generally one of the easiest way to reduce a page weight, and as a result, the page load time. Don't use Photoshop or other image editing tools, they're not very good for optimization. Use specialized tools such as Kraken.io or the excellent ImageOptim on Mac. For SVG images, you can use SVGOMG

The tools in use in YellowLabTools are not set to their maximum optimization power (JPEG quality 85), so you might be able to compress even more!

Please note that Yellow Lab Tools' engine (PhantomJS) is not compatible with image srcset (unless you use a polyfill). This can lead to incorrect page weight.

", "isOkThreshold": 10240, @@ -919,7 +919,7 @@ var policies = { "unit": 'bytes' }, "gzipCompression": { - "tool": "weightChecker", + "tool": "redownload", "label": "Gzip compression", "message": "

Measures the number of bytes that could be saved by compressing file transfers.

Gzip is a powerfull weight reducer and should be enabled on text-based assets in your server's configuration. Note that gzipping small files (< 1 KB) is arguable, and that some assets such as images should not be gzipped as they are already compressed. Here is a list of Content-Types that should be gzipped.

", "isOkThreshold": 5125, @@ -929,7 +929,7 @@ var policies = { "unit": 'bytes' }, "fileMinification": { - "tool": "weightChecker", + "tool": "redownload", "label": "File minification", "message": "

This is the weight that could be saved if all text resources were correctly minified.

The tools in use here are UglifyJS, clean-css and HTMLMinifier. These tools are so good that some of your minified files can be marked as unminified. Change your tool it this happens :)

The gains of minification are generally small, but the impact can be high when these text files are loaded on the critical path.

", "isOkThreshold": 5125, @@ -939,7 +939,7 @@ var policies = { "unit": 'bytes' }, "totalRequests": { - "tool": "weightChecker", + "tool": "redownload", "label": "Requests number", "message": "

This is one of the most important performance rule. Every request is slowing down the page loading.

There are several technics to reduce their number:

", "isOkThreshold": 15, @@ -1003,8 +1003,17 @@ var policies = { "isAbnormalThreshold": 10, "hasOffenders": true }, + "emptyRequests": { + "tool": "redownload", + "label": "Empty requests", + "message": "

List of GET requests that respond with an empty body. These are probably the easiest requests to remove.

", + "isOkThreshold": 0, + "isBadThreshold": 1, + "isAbnormalThreshold": 5, + "hasOffenders": true + }, "smallRequests": { - "tool": "weightChecker", + "tool": "redownload", "label": "Small requests", "message": "

List of all requests that are less than 2 KB. Try to merge them with other files.

", "isOkThreshold": 4, diff --git a/lib/metadata/scoreProfileGeneric.json b/lib/metadata/scoreProfileGeneric.json index 565ead7..ccd5512 100644 --- a/lib/metadata/scoreProfileGeneric.json +++ b/lib/metadata/scoreProfileGeneric.json @@ -16,6 +16,7 @@ "domains": 3, "notFound": 3, "multipleRequests": 2, + "emptyRequests": 3, "smallRequests": 1, "lazyLoadableImagesBelowTheFold": 2, "hiddenImages": 1 diff --git a/lib/runner.js b/lib/runner.js index 4c5daf2..3648a73 100644 --- a/lib/runner.js +++ b/lib/runner.js @@ -6,7 +6,7 @@ var jsExecutionTransformer = require('./tools/jsExecutionTransformer'); var colorDiff = require('./tools/colorDiff'); var mediaQueriesChecker = require('./tools/mediaQueriesChecker'); var isHttp2 = require('./tools/isHttp2'); -var weightChecker = require('./tools/weightChecker/weightChecker'); +var redownload = require('./tools/redownload/redownload'); var rulesChecker = require('./rulesChecker'); var scoreCalculator = require('./scoreCalculator'); @@ -38,7 +38,7 @@ var Runner = function(params) { data = mediaQueriesChecker.analyzeMediaQueries(data); // Redownload every file - return weightChecker.recheckAllFiles(data); + return redownload.recheckAllFiles(data); }) diff --git a/lib/tools/weightChecker/contentTypeChecker.js b/lib/tools/redownload/contentTypeChecker.js similarity index 100% rename from lib/tools/weightChecker/contentTypeChecker.js rename to lib/tools/redownload/contentTypeChecker.js diff --git a/lib/tools/weightChecker/fileMinifier.js b/lib/tools/redownload/fileMinifier.js similarity index 100% rename from lib/tools/weightChecker/fileMinifier.js rename to lib/tools/redownload/fileMinifier.js diff --git a/lib/tools/weightChecker/gzipCompressor.js b/lib/tools/redownload/gzipCompressor.js similarity index 100% rename from lib/tools/weightChecker/gzipCompressor.js rename to lib/tools/redownload/gzipCompressor.js diff --git a/lib/tools/weightChecker/imageOptimizer.js b/lib/tools/redownload/imageOptimizer.js similarity index 100% rename from lib/tools/weightChecker/imageOptimizer.js rename to lib/tools/redownload/imageOptimizer.js diff --git a/lib/tools/weightChecker/weightChecker.js b/lib/tools/redownload/redownload.js similarity index 94% rename from lib/tools/weightChecker/weightChecker.js rename to lib/tools/redownload/redownload.js index f69fc4a..693eb23 100644 --- a/lib/tools/weightChecker/weightChecker.js +++ b/lib/tools/redownload/redownload.js @@ -5,7 +5,7 @@ */ -var debug = require('debug')('ylt:weightChecker'); +var debug = require('debug')('ylt:redownload'); var Q = require('q'); var http = require('http'); var zlib = require('zlib'); @@ -18,7 +18,7 @@ var gzipCompressor = require('./gzipCompressor'); var contentTypeChecker = require('./contentTypeChecker'); -var WeightChecker = function() { +var Redownload = function() { var MAX_PARALLEL_DOWNLOADS = 10; var REQUEST_TIMEOUT = 15000; // 15 seconds @@ -86,9 +86,10 @@ var WeightChecker = function() { offenders.totalRequests = listRequestsByType(results); metrics.totalRequests = offenders.totalRequests.total; - // Remove unwanted requests (redirections, about:blank) + + // Remove unwanted responses (redownload failed, about:blank) results = results.filter(function(result) { - return (result !== null && result.weightCheck && result.weightCheck.bodySize > 0); + return (result !== null && result.weightCheck && result.url.indexOf('about:blank') !== 0); }); @@ -100,6 +101,17 @@ var WeightChecker = function() { offenders.totalWeight = listRequestWeight(results); metrics.totalWeight = offenders.totalWeight.totalWeight; + // Empty files + offenders.emptyRequests = listEmptyRequests(results); + metrics.emptyRequests = offenders.emptyRequests.length; + + + // Now emove unwanted responses (redirections) + results = results.filter(function(result) { + return (result.status < 300 || result.status >= 400); + }); + + // Image compression offenders.imageOptimization = listImageNotOptimized(results); metrics.imageOptimization = offenders.imageOptimization.totalGain; @@ -116,7 +128,7 @@ var WeightChecker = function() { offenders.smallRequests = listSmallRequests(results); metrics.smallRequests = offenders.smallRequests.total; - data.toolsResults.weightChecker = { + data.toolsResults.redownload = { metrics: metrics, offenders: offenders }; @@ -144,6 +156,20 @@ var WeightChecker = function() { return results; } + function listEmptyRequests(requests) { + var results = []; + + requests.forEach(function(req) { + var weight = req.weightCheck.bodySize || req.contentLength; + + if (weight === 0 && req.method === 'GET' && req.status === 200) { + results.push(req.url); + } + }); + + return results; + } + function listRequestWeight(requests) { var results = { totalWeight: 0, @@ -555,4 +581,4 @@ var WeightChecker = function() { }; }; -module.exports = new WeightChecker(); \ No newline at end of file +module.exports = new Redownload(); \ No newline at end of file diff --git a/test/core/contentTypeCheckerTest.js b/test/core/contentTypeCheckerTest.js index ae7c833..ac67cc4 100644 --- a/test/core/contentTypeCheckerTest.js +++ b/test/core/contentTypeCheckerTest.js @@ -1,5 +1,5 @@ var should = require('chai').should(); -var contentTypeChecker = require('../../lib/tools/weightChecker/contentTypeChecker'); +var contentTypeChecker = require('../../lib/tools/redownload/contentTypeChecker'); var fs = require('fs'); var path = require('path'); diff --git a/test/core/fileMinifierTest.js b/test/core/fileMinifierTest.js index 90ce452..348810b 100644 --- a/test/core/fileMinifierTest.js +++ b/test/core/fileMinifierTest.js @@ -1,5 +1,5 @@ var should = require('chai').should(); -var fileMinifier = require('../../lib/tools/weightChecker/fileMinifier'); +var fileMinifier = require('../../lib/tools/redownload/fileMinifier'); var fs = require('fs'); var path = require('path'); diff --git a/test/core/gzipCompressorTest.js b/test/core/gzipCompressorTest.js index 66a66d6..c436e02 100644 --- a/test/core/gzipCompressorTest.js +++ b/test/core/gzipCompressorTest.js @@ -1,6 +1,6 @@ var should = require('chai').should(); -var gzipCompressor = require('../../lib/tools/weightChecker/gzipCompressor'); -var fileMinifier = require('../../lib/tools/weightChecker/fileMinifier'); +var gzipCompressor = require('../../lib/tools/redownload/gzipCompressor'); +var fileMinifier = require('../../lib/tools/redownload/fileMinifier'); var fs = require('fs'); var path = require('path'); diff --git a/test/core/imageOptimizerTest.js b/test/core/imageOptimizerTest.js index 9871491..e858962 100644 --- a/test/core/imageOptimizerTest.js +++ b/test/core/imageOptimizerTest.js @@ -1,5 +1,5 @@ var should = require('chai').should(); -var imageOptimizer = require('../../lib/tools/weightChecker/imageOptimizer'); +var imageOptimizer = require('../../lib/tools/redownload/imageOptimizer'); var fs = require('fs'); var path = require('path'); diff --git a/test/core/weightCheckerTest.js b/test/core/redownloadTest.js similarity index 70% rename from test/core/weightCheckerTest.js rename to test/core/redownloadTest.js index 2e93bea..8a12ab3 100644 --- a/test/core/weightCheckerTest.js +++ b/test/core/redownloadTest.js @@ -1,9 +1,9 @@ var should = require('chai').should(); -var weightChecker = require('../../lib/tools/weightChecker/weightChecker'); +var redownload = require('../../lib/tools/redownload/redownload'); var fs = require('fs'); var path = require('path'); -describe('weightChecker', function() { +describe('redownload', function() { it('should download a list of files', function(done) { this.timeout(10000); @@ -121,49 +121,49 @@ describe('weightChecker', function() { } }; - weightChecker.recheckAllFiles(data) + redownload.recheckAllFiles(data) .then(function(data) { - data.toolsResults.should.have.a.property('weightChecker'); - data.toolsResults.weightChecker.should.have.a.property('metrics'); - data.toolsResults.weightChecker.should.have.a.property('offenders'); + data.toolsResults.should.have.a.property('redownload'); + data.toolsResults.redownload.should.have.a.property('metrics'); + data.toolsResults.redownload.should.have.a.property('offenders'); - data.toolsResults.weightChecker.offenders.should.have.a.property('totalWeight'); - data.toolsResults.weightChecker.offenders.totalWeight.totalWeight.should.be.above(0); - data.toolsResults.weightChecker.offenders.totalWeight.byType.html.requests.length.should.equal(1); - data.toolsResults.weightChecker.offenders.totalWeight.byType.js.requests.length.should.equal(2); - data.toolsResults.weightChecker.offenders.totalWeight.byType.css.requests.length.should.equal(1); - data.toolsResults.weightChecker.offenders.totalWeight.byType.image.requests.length.should.equal(2); - data.toolsResults.weightChecker.offenders.totalWeight.byType.other.requests.length.should.equal(1); + data.toolsResults.redownload.offenders.should.have.a.property('totalWeight'); + data.toolsResults.redownload.offenders.totalWeight.totalWeight.should.be.above(0); + data.toolsResults.redownload.offenders.totalWeight.byType.html.requests.length.should.equal(1); + data.toolsResults.redownload.offenders.totalWeight.byType.js.requests.length.should.equal(2); + data.toolsResults.redownload.offenders.totalWeight.byType.css.requests.length.should.equal(1); + data.toolsResults.redownload.offenders.totalWeight.byType.image.requests.length.should.equal(2); + data.toolsResults.redownload.offenders.totalWeight.byType.other.requests.length.should.equal(1); - data.toolsResults.weightChecker.offenders.should.have.a.property('imageOptimization'); - data.toolsResults.weightChecker.offenders.imageOptimization.totalGain.should.be.above(0); - data.toolsResults.weightChecker.offenders.imageOptimization.images.length.should.equal(2); + data.toolsResults.redownload.offenders.should.have.a.property('imageOptimization'); + data.toolsResults.redownload.offenders.imageOptimization.totalGain.should.be.above(0); + data.toolsResults.redownload.offenders.imageOptimization.images.length.should.equal(2); - data.toolsResults.weightChecker.offenders.should.have.a.property('gzipCompression'); - data.toolsResults.weightChecker.offenders.gzipCompression.totalGain.should.be.above(0); - data.toolsResults.weightChecker.offenders.gzipCompression.files.length.should.equal(5); + data.toolsResults.redownload.offenders.should.have.a.property('gzipCompression'); + data.toolsResults.redownload.offenders.gzipCompression.totalGain.should.be.above(0); + data.toolsResults.redownload.offenders.gzipCompression.files.length.should.equal(5); - data.toolsResults.weightChecker.offenders.should.have.a.property('fileMinification'); - data.toolsResults.weightChecker.offenders.fileMinification.totalGain.should.be.above(0); - data.toolsResults.weightChecker.offenders.fileMinification.files.length.should.equal(2); + data.toolsResults.redownload.offenders.should.have.a.property('fileMinification'); + data.toolsResults.redownload.offenders.fileMinification.totalGain.should.be.above(0); + data.toolsResults.redownload.offenders.fileMinification.files.length.should.equal(2); - data.toolsResults.weightChecker.metrics.should.have.a.property('totalRequests').that.equals(7); - data.toolsResults.weightChecker.offenders.should.have.a.property('totalRequests'); - data.toolsResults.weightChecker.offenders.totalRequests.byType.html.length.should.equal(1); - data.toolsResults.weightChecker.offenders.totalRequests.byType.js.length.should.equal(2); - data.toolsResults.weightChecker.offenders.totalRequests.byType.css.length.should.equal(1); - data.toolsResults.weightChecker.offenders.totalRequests.byType.image.length.should.equal(2); - data.toolsResults.weightChecker.offenders.totalRequests.byType.json.length.should.equal(0); - data.toolsResults.weightChecker.offenders.totalRequests.byType.webfont.length.should.equal(0); - data.toolsResults.weightChecker.offenders.totalRequests.byType.video.length.should.equal(0); - data.toolsResults.weightChecker.offenders.totalRequests.byType.other.length.should.equal(1); + data.toolsResults.redownload.metrics.should.have.a.property('totalRequests').that.equals(7); + data.toolsResults.redownload.offenders.should.have.a.property('totalRequests'); + data.toolsResults.redownload.offenders.totalRequests.byType.html.length.should.equal(1); + data.toolsResults.redownload.offenders.totalRequests.byType.js.length.should.equal(2); + data.toolsResults.redownload.offenders.totalRequests.byType.css.length.should.equal(1); + data.toolsResults.redownload.offenders.totalRequests.byType.image.length.should.equal(2); + data.toolsResults.redownload.offenders.totalRequests.byType.json.length.should.equal(0); + data.toolsResults.redownload.offenders.totalRequests.byType.webfont.length.should.equal(0); + data.toolsResults.redownload.offenders.totalRequests.byType.video.length.should.equal(0); + data.toolsResults.redownload.offenders.totalRequests.byType.other.length.should.equal(1); - data.toolsResults.weightChecker.metrics.should.have.a.property('smallRequests').that.equals(0); - data.toolsResults.weightChecker.offenders.should.have.a.property('smallRequests'); - data.toolsResults.weightChecker.offenders.smallRequests.byType.js.length.should.equal(0); - data.toolsResults.weightChecker.offenders.smallRequests.byType.css.length.should.equal(0); - data.toolsResults.weightChecker.offenders.smallRequests.byType.image.length.should.equal(0); + data.toolsResults.redownload.metrics.should.have.a.property('smallRequests').that.equals(0); + data.toolsResults.redownload.offenders.should.have.a.property('smallRequests'); + data.toolsResults.redownload.offenders.smallRequests.byType.js.length.should.equal(0); + data.toolsResults.redownload.offenders.smallRequests.byType.css.length.should.equal(0); + data.toolsResults.redownload.offenders.smallRequests.byType.image.length.should.equal(0); done(); }) @@ -187,7 +187,7 @@ describe('weightChecker', function() { type: 'js' }; - weightChecker.redownloadEntry(entry) + redownload.redownloadEntry(entry) .then(function(newEntry) { @@ -221,7 +221,7 @@ describe('weightChecker', function() { contentType: 'image/png' }; - weightChecker.redownloadEntry(entry) + redownload.redownloadEntry(entry) .then(function(newEntry) { @@ -258,7 +258,7 @@ describe('weightChecker', function() { contentLength: 999 }; - weightChecker.redownloadEntry(entry) + redownload.redownloadEntry(entry) .then(function(newEntry) { newEntry.weightCheck.should.have.a.property('message').that.equals('error while downloading: 404'); @@ -285,7 +285,7 @@ describe('weightChecker', function() { contentLength: 999 }; - weightChecker.redownloadEntry(entry) + redownload.redownloadEntry(entry) .then(function(newEntry) { newEntry.should.not.have.a.property('weightCheck'); @@ -298,7 +298,7 @@ describe('weightChecker', function() { }); it('should listRequestWeight', function() { - var totalWeightObj = weightChecker.listRequestWeight([{ + var totalWeightObj = redownload.listRequestWeight([{ method: 'GET', url: 'http://localhost:8388/jquery1.8.3.js', requestHeaders: { @@ -328,7 +328,7 @@ describe('weightChecker', function() { }); it('should listRequestWeight even if download failed', function() { - var totalWeightObj = weightChecker.listRequestWeight([{ + var totalWeightObj = redownload.listRequestWeight([{ method: 'GET', url: 'http://localhost:8388/jquery1.8.3.js', requestHeaders: {