The weight is of course very important if you want the page to load fast. Try to stay under 1MB, which is already very long to download over a slow connection.
", + "message": "The weight is of course very important if you want the page to load fast. Try to stay under 1.5MB.
", "isOkThreshold": 1572864, "isBadThreshold": 3145728, "isAbnormalThreshold": 5242880, @@ -746,6 +746,16 @@ var policies = { "hasOffenders": true, "unit": 'bytes' }, + "oldImageFormats": { + "tool": "redownload", + "label": "Old image formats", + "message": "Measures the number of bytes that could be saved by converting images to newer and more efficient formats. The current best image format for the web is AVIF and the second best is WebP.
Be careful, you need to provide a fallback for old browsers that don't support them (for example, Microsoft Edge doesn't support AVIF on Windows 10 or earlier) and search engine bots.
", + "isOkThreshold": 30720, + "isBadThreshold": 307200, + "isAbnormalThreshold": 512000, + "hasOffenders": true, + "unit": 'bytes' + }, "imagesTooLarge": { "tool": "redownload", "label": "Oversized images", diff --git a/lib/metadata/scoreProfileGeneric.json b/lib/metadata/scoreProfileGeneric.json index 8bff9e4..4ecdc42 100644 --- a/lib/metadata/scoreProfileGeneric.json +++ b/lib/metadata/scoreProfileGeneric.json @@ -5,6 +5,7 @@ "policies": { "totalWeight": 5, "imageOptimization": 2, + "oldImageFormats": 2, "imagesTooLarge": 1, "compression": 2, "fileMinification": 2 diff --git a/lib/screenshotHandler.js b/lib/screenshotHandler.js index 02875ac..c175fd1 100644 --- a/lib/screenshotHandler.js +++ b/lib/screenshotHandler.js @@ -4,6 +4,8 @@ var Q = require('q'); var fs = require('fs'); var path = require('path'); +// Disable sharp cache to reduce the "disk is full" error on Amazon Lambda +sharp.cache(false); var screenshotHandler = function() { diff --git a/lib/tools/redownload/contentTypeChecker.js b/lib/tools/redownload/contentTypeChecker.js index 08f5b23..a37c7d5 100644 --- a/lib/tools/redownload/contentTypeChecker.js +++ b/lib/tools/redownload/contentTypeChecker.js @@ -52,7 +52,7 @@ var ContentTypeChecker = function() { // If it's an image or a font, then rewrite. if (foundType !== null && (foundType.type === 'image' || foundType.type === 'webfont' || foundType.type === 'json')) { if (foundType.type !== entry.type) { - debug('Content type %s is wrong for %s. It should be %s.', entry.type, entry.ulr, foundType.type); + debug('Content type %s is wrong for %s. It should be %s.', entry.type, entry.url, foundType.type); } rewriteContentType(entry, foundType); } diff --git a/lib/tools/redownload/imageReformater.js b/lib/tools/redownload/imageReformater.js new file mode 100644 index 0000000..c882c6c --- /dev/null +++ b/lib/tools/redownload/imageReformater.js @@ -0,0 +1,138 @@ +var debug = require('debug')('ylt:imageReformater'); +var sharp = require('sharp'); + +// Disable sharp cache to reduce the "disk is full" error on Amazon Lambda +sharp.cache(false); + +var ImageOptimizer = function() { + + // https://www.industrialempathy.com/posts/avif-webp-quality-settings + const WEBP_QUALITY = 82; + const AVIF_QUALITY = 64; + + async function reformatImage(entry) { + if (!entry.weightCheck || !entry.weightCheck.bodyBuffer) { + // No valid file available + return entry; + } + + var fileSize = entry.weightCheck.uncompressedSize; + debug('Let\'s try to convert %s to other image formats', entry.url); + debug('Current file size is %d', fileSize); + + var animated = await isAnimated(entry); + debug('Check if the file is animated: %s', animated); + + + if (isJPEG(entry) || isPNG(entry)) { + debug('File is %s, let\'s try to convert it to WebP', entry.contentType); + + try { + + const webpFile = await convertToWebp(entry.weightCheck.bodyBuffer, animated); + + if (webpFile) { + var webpFileSize = webpFile.length; + + debug('WebP transformation complete for %s', entry.url); + debug('WebP size is %d bytes', webpFileSize); + + if (webpFile.length > 0 && gainIsEnough(fileSize, webpFileSize)) { + entry.weightCheck.webpSize = webpFileSize; + debug('WebP size is %d bytes smaller (-%d%)', fileSize - webpFileSize, Math.round((fileSize - webpFileSize) * 100 / fileSize)); + } + + } else { + debug('Convertion to WebP didn\'t work'); + } + + } catch(err) { + debug('Error while converting to WebP, ignoring'); + } + } + + if (!animated && (isJPEG(entry) || isPNG(entry) || isWebP(entry))) { + debug('File is %s and is not animated, let\'s try to convert it to AVIF', entry.contentType); + + try { + + const avifFile = await convertToAvif(entry.weightCheck.bodyBuffer); + + if (avifFile) { + var avifFileSize = avifFile.length; + + debug('AVIF transformation complete for %s', entry.url); + debug('AVIF size is %d bytes', avifFileSize); + + if (avifFile.length > 0 && gainIsEnough(fileSize, avifFileSize)) { + entry.weightCheck.avifSize = avifFileSize; + debug('AVIF size is %d bytes smaller (-%d%)', fileSize - avifFileSize, Math.round((fileSize - avifFileSize) * 100 / fileSize)); + } + + } else { + debug('Convertion to AVIF didn\'t work'); + } + + } catch(err) { + debug('Error while converting to AVIF, ignoring'); + } + } + + return entry; + } + + async function convertToWebp(bodyBuffer, isAnimated) { + return sharp(bodyBuffer, {animated: isAnimated}) + .webp({quality: WEBP_QUALITY, alphaQuality: WEBP_QUALITY}) + .toBuffer(); + } + + async function convertToAvif(bodyBuffer) { + return sharp(bodyBuffer) + .webp({quality: AVIF_QUALITY}) + .toBuffer(); + } + + // The gain is estimated of enough value if it's over 2KB or over 20%, + // but it's ignored if is below 100 bytes + function gainIsEnough(oldWeight, newWeight) { + var gain = oldWeight - newWeight; + var ratio = gain / oldWeight; + return (gain > 2048 || (ratio > 0.2 && gain > 100)); + } + + function isJPEG(entry) { + return entry.isImage && entry.contentType === 'image/jpeg'; + } + + function isPNG(entry) { + return entry.isImage && entry.contentType === 'image/png'; + } + + function isWebP(entry) { + return entry.isImage && entry.contentType === 'image/webp'; + } + + function entryTypeCanBeReformated(entry) { + return isJPEG(entry) || isPNG(entry) || isWebP(entry); + } + + async function isAnimated(entry) { + if (isWebP(entry)) { + const metadata = await sharp(entry.weightCheck.bodyBuffer).metadata(); + return metadata.pages > 1; + } + return false; + } + + return { + reformatImage: reformatImage, + convertToWebp: convertToWebp, + convertToAvif: convertToAvif, + gainIsEnough: gainIsEnough, + entryTypeCanBeReformated: entryTypeCanBeReformated, + isAnimated: isAnimated + }; +}; + +module.exports = new ImageOptimizer(); \ No newline at end of file diff --git a/lib/tools/redownload/redownload.js b/lib/tools/redownload/redownload.js index 803b0dd..e7d8f26 100644 --- a/lib/tools/redownload/redownload.js +++ b/lib/tools/redownload/redownload.js @@ -15,6 +15,7 @@ var request = require('request'); var md5 = require('md5'); var imageOptimizer = require('./imageOptimizer'); +var imageReformater = require('./imageReformater'); var fileMinifier = require('./fileMinifier'); var gzipCompressor = require('./gzipCompressor'); var brotliCompressor = require('./brotliCompressor'); @@ -78,6 +79,10 @@ var Redownload = function() { .then(imageOptimizer.optimizeImage) + .then(function(entry) { + return Q(imageReformater.reformatImage(entry)); + }) + .then(imageDimensions.getDimensions) .then(fileMinifier.minifyFile) @@ -91,7 +96,7 @@ var Redownload = function() { }) .then(function(newEntry) { - debug('File %s - Redownloaded, optimized, minified, compressed, analyzed: done', entry.url); + debug('File %s - Redownloaded, optimized, reformated, minified, compressed, analyzed: done', entry.url); // For the progress bar doneCount ++; @@ -154,6 +159,10 @@ var Redownload = function() { offenders.imageOptimization = listImagesNotOptimized(results); metrics.imageOptimization = offenders.imageOptimization.totalGain; + // Old image formats + offenders.oldImageFormats = listImagesWithOldFormats(results); + metrics.oldImageFormats = offenders.oldImageFormats.totalGain; + // Image width offenders.imagesTooLarge = listImagesTooLarge(results, data.params.options.device); metrics.imagesTooLarge = offenders.imagesTooLarge.length; @@ -398,6 +407,41 @@ var Redownload = function() { return results; } + function listImagesWithOldFormats(requests) { + var results = { + totalGain: 0, + images: [] + }; + + requests.forEach(function(req) { + + if (req.weightCheck.bodySize > 0 && + imageReformater.entryTypeCanBeReformated(req) && + (req.weightCheck.webpSize > 0 || req.weightCheck.avifSize > 0)) { + + var image = { + url: req.url, + originalWeigth: req.weightCheck.bodySize + }; + + if (req.weightCheck.webpSize) { + image.webpSize = req.weightCheck.webpSize; + } + if (req.weightCheck.avifSize) { + image.avifSize = req.weightCheck.avifSize; + } + + var smallest = Math.min(image.webpSize || Infinity, image.avifSize || Infinity); + image.gain = req.weightCheck.bodySize - smallest; + + results.totalGain += image.gain; + results.images.push(image); + } + }); + + return results; + } + function listImagesTooLarge(requests, device) { var results = []; diff --git a/test/core/imageReformaterTest.js b/test/core/imageReformaterTest.js new file mode 100644 index 0000000..07d6541 --- /dev/null +++ b/test/core/imageReformaterTest.js @@ -0,0 +1,122 @@ +var should = require('chai').should(); +var imageReformater = require('../../lib/tools/redownload/imageReformater'); +var fs = require('fs'); +var path = require('path'); + +describe('imageReformater', function() { + + it('should convert a JPEG image to WebP and AVIF', async function() { + var fileContent = fs.readFileSync(path.resolve(__dirname, '../www/jpeg-image.jpg')); + let entry = { + isImage: true, + type: 'image', + contentType: 'image/jpeg', + weightCheck: { + bodyBuffer: fileContent, + uncompressedSize: fileContent.length + } + }; + + var newEntry = await imageReformater.reformatImage(entry); + + newEntry.weightCheck.should.have.a.property('webpSize'); + newEntry.weightCheck.webpSize.should.be.below(fileContent.length); + + newEntry.weightCheck.should.have.a.property('avifSize'); + newEntry.weightCheck.avifSize.should.be.below(fileContent.length); + }); + + it('should convert a PNG image to WebP and AVIF', async function() { + var fileContent = fs.readFileSync(path.resolve(__dirname, '../www/jpeg-image.jpg')); + let entry = { + isImage: true, + type: 'image', + contentType: 'image/png', + weightCheck: { + bodyBuffer: fileContent, + uncompressedSize: fileContent.length + } + }; + + var newEntry = await imageReformater.reformatImage(entry); + + newEntry.weightCheck.should.have.a.property('webpSize'); + newEntry.weightCheck.webpSize.should.be.below(fileContent.length); + + newEntry.weightCheck.should.have.a.property('avifSize'); + newEntry.weightCheck.avifSize.should.be.below(fileContent.length); + }); + + it('should convert a WebP image to AVIF', async function() { + var fileContent = fs.readFileSync(path.resolve(__dirname, '../www/jpeg-image.jpg')); + let entry = { + isImage: true, + type: 'image', + contentType: 'image/webp', + weightCheck: { + bodyBuffer: fileContent, + uncompressedSize: fileContent.length + } + }; + + var newEntry = await imageReformater.reformatImage(entry); + + newEntry.weightCheck.should.not.have.a.property('webpSize'); + + newEntry.weightCheck.should.have.a.property('avifSize'); + newEntry.weightCheck.avifSize.should.be.below(fileContent.length); + }); + + it('should recognize an animated WebP', async function() { + // Test on an animated image + let fileContent = fs.readFileSync(path.resolve(__dirname, '../www/animated.webp')); + let entry = { + isImage: true, + type: 'image', + contentType: 'image/webp', + weightCheck: { + bodyBuffer: fileContent, + uncompressedSize: fileContent.length + } + }; + + (await imageReformater.isAnimated(entry)).should.equal(true); + + // Test on a not animated image + fileContent = fs.readFileSync(path.resolve(__dirname, '../www/monster.webp')); + entry.weightCheck.bodyBuffer = fileContent; + (await imageReformater.isAnimated(entry)).should.equal(false); + }); + + it('should not convert an animated WebP', async function() { + // Test on an animated image + let fileContent = fs.readFileSync(path.resolve(__dirname, '../www/animated.webp')); + let entry = { + isImage: true, + type: 'image', + contentType: 'image/webp', + weightCheck: { + bodyBuffer: fileContent, + uncompressedSize: fileContent.length + } + }; + + var newEntry = await imageReformater.reformatImage(entry); + + // Test on a not animated image + newEntry.weightCheck.should.not.have.a.property('avifSize'); + }); + + it('should determine if gain is enough', function() { + imageReformater.gainIsEnough(20000, 10000).should.equal(true); + imageReformater.gainIsEnough(2000, 1000).should.equal(true); + imageReformater.gainIsEnough(20000, 21000).should.equal(false); + imageReformater.gainIsEnough(20000, 40000).should.equal(false); + imageReformater.gainIsEnough(20000, 19500).should.equal(false); + imageReformater.gainIsEnough(250, 120).should.equal(true); + imageReformater.gainIsEnough(200, 120).should.equal(false); + imageReformater.gainIsEnough(2000, 1900).should.equal(false); + imageReformater.gainIsEnough(200000, 197000).should.equal(true); + }); + +}); diff --git a/test/core/redownloadTest.js b/test/core/redownloadTest.js index 0c02dae..ada3e22 100644 --- a/test/core/redownloadTest.js +++ b/test/core/redownloadTest.js @@ -86,6 +86,10 @@ describe('redownload', function() { data.toolsResults.redownload.offenders.imageOptimization.totalGain.should.be.above(0); data.toolsResults.redownload.offenders.imageOptimization.images.length.should.equal(2); + data.toolsResults.redownload.offenders.should.have.a.property('oldImageFormats'); + data.toolsResults.redownload.offenders.oldImageFormats.totalGain.should.be.above(0); + data.toolsResults.redownload.offenders.oldImageFormats.images.length.should.equal(1); + data.toolsResults.redownload.offenders.should.have.a.property('imagesTooLarge'); data.toolsResults.redownload.offenders.imagesTooLarge.length.should.equal(0); @@ -167,18 +171,9 @@ describe('redownload', function() { redownload.redownloadEntry(entry) .then(function(newEntry) { - newEntry.weightCheck.bodySize.should.equal(4193); newEntry.weightCheck.bodyBuffer.should.deep.equal(fileContent); - - // Opening the image in jimp to check if the format is good - var Jimp = require('jimp'); - Jimp.read(newEntry.weightCheck.bodyBuffer, function(err, image) { - image.bitmap.width.should.equal(620); - image.bitmap.height.should.equal(104); - done(err); - }); - + done(); }) .fail(function(err) { diff --git a/test/www/animated.webp b/test/www/animated.webp new file mode 100644 index 0000000..7a1d3fe Binary files /dev/null and b/test/www/animated.webp differ diff --git a/test/www/monster.webp b/test/www/monster.webp new file mode 100644 index 0000000..e6ec320 Binary files /dev/null and b/test/www/monster.webp differ