WEBDIENSTE/YellowLabTools
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add Brotli compression as the default compresion

Browse Source
This commit is contained in:
Gaël Métais
2020-12-21 10:21:20 +01:00
parent be5549bf81
commit 51dc1b1618
10 changed files with 234 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/metadata/policies.js
+7 -7
View File
@@ -786,13 +786,13 @@ var policies = {
"isAbnormalThreshold": 10,
"hasOffenders": true
},
"gzipCompression": {
"compression": {
"tool": "redownload",
"label": "Gzip compression",
"message": "<p>Measures the number of bytes that could be saved by compressing file transfers.</p><p>Gzip is a powerfull weight reducer and should be enabled on text-based assets in your server's configuration. Note that gzipping small files (< 1 KB) is arguable, and that some assets such as images should not be gzipped as they are already compressed. <a href=\"https://gist.github.com/gmetais/971ce13a1fbeebd88445\" target=\"_blank\">Here</a> is a list of Content-Types that should be gzipped.</p>",
"isOkThreshold": 5125,
"isBadThreshold": 81920,
"isAbnormalThreshold": 153600,
"label": "Gzip/Brotli compression",
"message": "<p>Measures the number of bytes that could be saved by compressing textual files.</p><p>Gzip is a good old weight reducer. But Brotli is a new-generation compression tool and it gives better results. All major server systems are now compatible with Brotli.</p><p>Note that compressing small files (< 1 KB) is arguable, and that some assets such as images should not be gzipped as they are already compressed. <a href=\"https://gist.github.com/gmetais/971ce13a1fbeebd88445\" target=\"_blank\">Here</a> is a list of Content-Types that should be compressed.</p>",
"isOkThreshold": 20480,
"isBadThreshold": 204800,
"isAbnormalThreshold": 409600,
"hasOffenders": true,
"unit": 'bytes'
},
@@ -932,7 +932,7 @@ var policies = {
"label": "Woff 2",
"message": "<p>The fonts listed here could be lighter if they were served with the latest woff 2 font file format. Some online tools can help you easily convert older formats to woff 2.</p>",
"isOkThreshold": 0,
"isBadThreshold": 61440,
"isBadThreshold": 51200,
"isAbnormalThreshold": 122880,
"hasOffenders": true,
"unit": 'bytes'
lib/metadata/scoreProfileGeneric.json
+2 -2
View File
@@ -5,8 +5,8 @@
"policies": {
"totalWeight": 5,
"imageOptimization": 2,
"imagesTooLarge": 2,
"gzipCompression": 2,
"imagesTooLarge": 1,
"compression": 2,
"fileMinification": 1
}
},
lib/tools/redownload/brotliCompressor.js
+115
View File
@@ -0,0 +1,115 @@
var debug = require('debug')('ylt:brotliCompressor');
var Q = require('q');
var brotli = require('brotli');
var gzipCompressor = require('./gzipCompressor');
var GzipCompressor = function() {
var BROTLI_COMPRESSION_LEVEL = 9;
function compressFile(entry) {
debug('Entering brotli compressor');
return brotlifyFile(entry)
.then(brotliOptimizedFile);
}
// Compress with Brotli files that were not compressed or not with brotli (with gzip or deflate)
function brotlifyFile(entry) {
var deferred = Q.defer();
if (gzipCompressor.entryTypeCanBeCompressed(entry) && entry.weightCheck && entry.weightCheck.bodyBuffer) {
if (!entry.weightCheck.isCompressed) {
debug('File %s was not compressed at all, trying Brotli over it.', entry.url);
} else if (entry.weightCheck.compressionTool !== 'brotli') {
debug('File %s was compressed with %s. Trying with Brotli.', entry.url, entry.weightCheck.compressionTool);
}
try {
var buffer = brotli.compress(entry.weightCheck.bodyBuffer, {
mode: 0,
quality: 9
});
if (buffer) {
var compressedSize = buffer.length;
if (!entry.weightCheck.isCompressed) {
debug('Brotli size is %d, was %d, this is %d% better.', compressedSize, entry.weightCheck.bodySize, Math.round((entry.weightCheck.bodySize - compressedSize) * 100 / entry.weightCheck.bodySize));
} else if (entry.weightCheck.compressionTool !== 'brotli') {
debug('Brotli size is %d, was %d with %s, this is %d% better.', compressedSize, entry.weightCheck.bodySize, entry.weightCheck.compressionTool, Math.round((entry.weightCheck.bodySize - compressedSize) * 100 / entry.weightCheck.bodySize));
}
entry.weightCheck.afterBrotliCompression = compressedSize;
} else {
debug('Failed to brotlify %s', entry.url);
}
deferred.resolve(entry);
} catch (err) {
debug(err);
deferred.reject(err);
}
} else {
debug('Compression not needed');
deferred.resolve(entry);
}
return deferred.promise;
}
// Gzip a file after minification or optimization if this step was successful
function brotliOptimizedFile(entry) {
var deferred = Q.defer();
if (gzipCompressor.entryTypeCanBeCompressed(entry) && entry.weightCheck && entry.weightCheck.isOptimized === false) {
debug('Trying to brotlify file after minification: %s', entry.url);
var uncompressedSize = entry.weightCheck.optimized;
try {
var buffer = brotli.compress(entry.weightCheck.bodyAfterOptimization, {
mode: 1,
quality: BROTLI_COMPRESSION_LEVEL
});
if (buffer) {
var compressedSize = buffer.length;
debug('Correctly brotlified the minified file, was %d and is now %d bytes', uncompressedSize, compressedSize);
entry.weightCheck.afterOptimizationAndBrotliCompression = compressedSize;
} else {
debug('Failed to brotlify %s', entry.url);
}
deferred.resolve(entry);
} catch (err) {
debug(err);
deferred.reject(err);
}
} else {
debug('Compressing optimized file not needed');
deferred.resolve(entry);
}
return deferred.promise;
}
return {
compressFile: compressFile
};
};
module.exports = new GzipCompressor();
lib/tools/redownload/contentTypeChecker.js
+4 -7
View File
@@ -56,15 +56,12 @@ var ContentTypeChecker = function() {
try {
foundType = findContentType(entry.weightCheck.bodyBuffer);
if (foundType !== null) {
if (foundType.type === 'webfont') {
// Always rewrite fonts for woff2 checking
rewriteContentType(entry, foundType);
} else if (foundType.type !== entry.type) {
// For other kind of files, just rewrite if needed
// If it's an image or a font, then rewrite.
if (foundType !== null && (foundType.type === 'image' || foundType.type === 'webfont')) {
if (foundType.type !== entry.type) {
debug('Content type %s is wrong for %s. It should be %s.', entry.type, entry.ulr, foundType.type);
rewriteContentType(entry, foundType);
}
rewriteContentType(entry, foundType);
}
} catch(err) {
lib/tools/redownload/fontAnalyzer.js
+1 -1
View File
@@ -125,7 +125,7 @@ var FontAnalyzer = function() {
name: font.fullName || font.postscriptName || font.familyName,
numGlyphs: font.numGlyphs,
averageGlyphComplexity: getAverageGlyphComplexity(font),
compressedWeight: entry.weightCheck.afterCompression || entry.weightCheck.bodySize,
compressedWeight: entry.weightCheck.afterGzipCompression || entry.weightCheck.bodySize,
unicodeRanges: readUnicodeRanges(font.characterSet, charsListOnPage),
numGlyphsInCommonWithPageContent: countPossiblyUsedGlyphs(getCharacterSetAsString(font.characterSet), charsListOnPage)
};
lib/tools/redownload/gzipCompressor.js
+8 -7
View File
@@ -6,7 +6,7 @@ var zlib = require('zlib');
var GzipCompressor = function() {
function compressFile(entry) {
debug('Entering compressFile');
debug('Entering gzip compressor');
return gzipUncompressedFile(entry)
.then(gzipOptimizedFile);
@@ -16,7 +16,7 @@ var GzipCompressor = function() {
function gzipUncompressedFile(entry) {
var deferred = Q.defer();
if (entryTypeCanBeGzipped(entry) && entry.weightCheck && !entry.weightCheck.isCompressed && entry.weightCheck.bodyBuffer) {
if (entryTypeCanBeCompressed(entry) && entry.weightCheck && !entry.weightCheck.isCompressed && entry.weightCheck.bodyBuffer) {
debug('Compression missing, trying to gzip file %s', entry.url);
var uncompressedSize = entry.weightCheck.uncompressedSize;
@@ -33,7 +33,7 @@ var GzipCompressor = function() {
if (gainIsEnough(uncompressedSize, compressedSize)) {
debug('File correctly gziped, was %d and is now %d bytes', uncompressedSize, compressedSize);
entry.weightCheck.afterCompression = compressedSize;
entry.weightCheck.afterGzipCompression = compressedSize;
} else {
debug('Gzip gain is not enough, was %d and is now %d bytes', uncompressedSize, compressedSize);
}
@@ -53,7 +53,7 @@ var GzipCompressor = function() {
function gzipOptimizedFile(entry) {
var deferred = Q.defer();
if (entryTypeCanBeGzipped(entry) && entry.weightCheck && entry.weightCheck.isOptimized === false) {
if (entryTypeCanBeCompressed(entry) && entry.weightCheck && entry.weightCheck.isOptimized === false) {
debug('Trying to gzip file after minification: %s', entry.url);
var uncompressedSize = entry.weightCheck.optimized;
@@ -68,7 +68,7 @@ var GzipCompressor = function() {
var compressedSize = buffer.length;
debug('Correctly gziped the minified file, was %d and is now %d bytes', uncompressedSize, compressedSize);
entry.weightCheck.afterOptimizationAndCompression = compressedSize;
entry.weightCheck.afterOptimizationAndGzipCompression = compressedSize;
deferred.resolve(entry);
}
@@ -89,13 +89,14 @@ var GzipCompressor = function() {
return (gain > 2048 || (ratio > 0.2 && gain > 100));
}
function entryTypeCanBeGzipped(entry) {
function entryTypeCanBeCompressed(entry) {
return entry.isJS || entry.isCSS || entry.isHTML || entry.isJSON || entry.isSVG || entry.isTTF || entry.isXML || entry.isFavicon;
}
return {
compressFile: compressFile,
entryTypeCanBeGzipped: entryTypeCanBeGzipped
gainIsEnough: gainIsEnough,
entryTypeCanBeCompressed: entryTypeCanBeCompressed
};
};
lib/tools/redownload/redownload.js
+43 -22
View File
@@ -17,6 +17,7 @@ var md5 = require('md5');
var imageOptimizer = require('./imageOptimizer');
var fileMinifier = require('./fileMinifier');
var gzipCompressor = require('./gzipCompressor');
var brotliCompressor = require('./brotliCompressor');
var contentTypeChecker = require('./contentTypeChecker');
var fontAnalyzer = require('./fontAnalyzer');
var imageDimensions = require('./imageDimensions');
@@ -80,6 +81,8 @@ var Redownload = function() {
.then(gzipCompressor.compressFile)
.then(brotliCompressor.compressFile)
.then(function(entry) {
return fontAnalyzer.analyzeFont(entry, differentCharacters);
})
@@ -151,9 +154,9 @@ var Redownload = function() {
offenders.fileMinification = listFilesNotMinified(results);
metrics.fileMinification = offenders.fileMinification.totalGain;
// Gzip compression
offenders.gzipCompression = listFilesNotGzipped(results);
metrics.gzipCompression = offenders.gzipCompression.totalGain;
// Gzip/Brotli compression
offenders.compression = listFilesNotBrotlified(results);
metrics.compression = offenders.compression.totalGain;
// Small requests
offenders.smallRequests = listSmallRequests(results);
@@ -338,8 +341,8 @@ var Redownload = function() {
requests.forEach(function(req) {
if (req.weightCheck.bodySize > 0 && imageOptimizer.entryTypeCanBeOptimized(req) && req.weightCheck.isOptimized === false) {
var before = req.weightCheck.afterCompression || req.weightCheck.bodySize;
var after = req.weightCheck.afterOptimizationAndCompression || req.weightCheck.optimized;
var before = req.weightCheck.afterGzipCompression || req.weightCheck.bodySize;
var after = req.weightCheck.afterOptimizationAndGzipCompression || req.weightCheck.optimized;
var gain = before - after;
if (gain > 200) {
@@ -349,8 +352,8 @@ var Redownload = function() {
url: req.url,
original: req.weightCheck.bodySize,
isCompressed: req.weightCheck.isCompressed,
afterCompression: req.weightCheck.afterCompression,
afterOptimizationAndCompression: req.weightCheck.afterOptimizationAndCompression,
afterGzipCompression: req.weightCheck.afterGzipCompression,
afterOptimizationAndGzipCompression: req.weightCheck.afterOptimizationAndGzipCompression,
lossless: req.weightCheck.lossless,
lossy: req.weightCheck.lossy,
gain: gain
@@ -398,8 +401,8 @@ var Redownload = function() {
requests.forEach(function(req) {
if (req.weightCheck.bodySize > 0 && fileMinifier.entryTypeCanBeMinified(req) && req.weightCheck.isOptimized === false) {
var before = req.weightCheck.afterCompression || req.weightCheck.bodySize;
var after = req.weightCheck.afterOptimizationAndCompression || req.weightCheck.optimized;
var before = req.weightCheck.afterGzipCompression || req.weightCheck.bodySize;
var after = req.weightCheck.afterOptimizationAndGzipCompression || req.weightCheck.optimized;
var gain = before - after;
if (gain > 200) {
@@ -409,8 +412,8 @@ var Redownload = function() {
url: req.url,
original: req.weightCheck.bodySize,
isCompressed: req.weightCheck.isCompressed,
afterCompression: req.weightCheck.afterCompression,
afterOptimizationAndCompression: req.weightCheck.afterOptimizationAndCompression,
afterGzipCompression: req.weightCheck.afterGzipCompression,
afterOptimizationAndGzipCompression: req.weightCheck.afterOptimizationAndGzipCompression,
optimized: req.weightCheck.optimized,
gain: gain
});
@@ -421,24 +424,37 @@ var Redownload = function() {
return results;
}
function listFilesNotGzipped(requests) {
function listFilesNotBrotlified(requests) {
var results = {
totalGain: 0,
files: []
};
requests.forEach(function(req) {
if (req.weightCheck.uncompressedSize && req.weightCheck.isCompressed === false && req.weightCheck.afterCompression) {
var gain = req.weightCheck.uncompressedSize - req.weightCheck.afterCompression;
results.totalGain += gain;
results.files.push({
if (req.weightCheck.compressionTool !== 'brotli') {
var file = {
url: req.url,
original: req.weightCheck.uncompressedSize,
gzipped: req.weightCheck.afterCompression,
gain: gain
});
wasCompressed: req.weightCheck.isCompressed,
brotlified: req.weightCheck.afterBrotliCompression
};
if (req.weightCheck.isCompressed) {
// The file was already gzipped (or deflated)
file.originalSize = req.weightCheck.bodySize;
file.gain = req.weightCheck.bodySize - req.weightCheck.afterBrotliCompression;
} else {
// The file was not compressed at all
file.originalSize = req.weightCheck.uncompressedSize;
file.gzipped = req.weightCheck.afterGzipCompression;
file.gain = req.weightCheck.uncompressedSize - req.weightCheck.afterBrotliCompression;
}
// Just checking a last time if the gain is positive
if (file.gain > 200) {
results.totalGain += file.gain;
results.files.push(file);
}
}
});
@@ -817,6 +833,7 @@ var Redownload = function() {
var bodySize = 0; // bytes size over the wire
var bodyChunks = []; // an array of buffers
var isCompressed = false;
var compressionTool = '';
function tally() {
@@ -832,6 +849,7 @@ var Redownload = function() {
headersSize: Buffer.byteLength(rawHeaders, 'utf8'),
bodySize: bodySize,
isCompressed: isCompressed,
compressionTool: compressionTool,
uncompressedSize: uncompressedSize
};
@@ -849,6 +867,7 @@ var Redownload = function() {
uncompressedSize += data.length;
}).on('end', function () {
isCompressed = true;
compressionTool = 'gzip';
tally();
}).on('error', function(err) {
debug('Error while decoding %s', requestOptions.url);
@@ -871,6 +890,7 @@ var Redownload = function() {
uncompressedSize += data.length;
}).on('end', function () {
isCompressed = true;
compressionTool = 'deflate';
tally();
}).on('error', function(err) {
debug('Error while decoding %s', requestOptions.url);
@@ -893,6 +913,7 @@ var Redownload = function() {
uncompressedSize += data.length;
}).on('end', function () {
isCompressed = true;
compressionTool = 'brotli';
tally();
}).on('error', function(err) {
debug('Error while decoding %s', requestOptions.url);