diff --git a/lib/index.js b/lib/index.js index 8285f33..68bf50c 100644 --- a/lib/index.js +++ b/lib/index.js @@ -33,11 +33,10 @@ var yellowLabTools = function(url, options) { // If a screenshot saveFunction was provided in the options if (options && typeof options.saveScreenshotFn === 'function') { - const screenshotTmpPath = data.params.options.screenshot; debug('Now optimizing screenshot...'); // TODO: temporarily set all screenshot sizes to 600px, until we find a solution - ScreenshotHandler.findAndOptimizeScreenshot(screenshotTmpPath, 600) + ScreenshotHandler.findAndOptimizeScreenshot(data.params.options.screenshot, 600) .then(function(screenshotBuffer) { debug('Screenshot optimized, now saving...'); @@ -51,7 +50,6 @@ var yellowLabTools = function(url, options) { // Remove uneeded temp screenshot path delete data.params.options.screenshot; - return ScreenshotHandler.deleteTmpFile(screenshotTmpPath); }) .catch(function(err) { diff --git a/lib/metadata/policies.js b/lib/metadata/policies.js index 7214982..3157fb1 100644 --- a/lib/metadata/policies.js +++ b/lib/metadata/policies.js @@ -729,7 +729,7 @@ var policies = { "totalWeight": { "tool": "redownload", "label": "Total weight", - "message": "

The weight is of course very important if you want the page to load fast. Try to stay under 1.5MB.

", + "message": "

The weight is of course very important if you want the page to load fast. Try to stay under 1MB, which is already very long to download over a slow connection.

", "isOkThreshold": 1572864, "isBadThreshold": 3145728, "isAbnormalThreshold": 5242880, @@ -746,16 +746,6 @@ var policies = { "hasOffenders": true, "unit": 'bytes' }, - "oldImageFormats": { - "tool": "redownload", - "label": "Old image formats", - "message": "

Measures the number of bytes that could be saved by converting images to newer and more efficient formats. The best image format is generally AVIF and the second best is WebP.

Be careful, you need to provide fallback images for old browsers and search engine bots.

", - "isOkThreshold": 30720, - "isBadThreshold": 307200, - "isAbnormalThreshold": 512000, - "hasOffenders": true, - "unit": 'bytes' - }, "imagesTooLarge": { "tool": "redownload", "label": "Oversized images", diff --git a/lib/metadata/scoreProfileGeneric.json b/lib/metadata/scoreProfileGeneric.json index 4ecdc42..8bff9e4 100644 --- a/lib/metadata/scoreProfileGeneric.json +++ b/lib/metadata/scoreProfileGeneric.json @@ -5,7 +5,6 @@ "policies": { "totalWeight": 5, "imageOptimization": 2, - "oldImageFormats": 2, "imagesTooLarge": 1, "compression": 2, "fileMinification": 2 diff --git a/lib/screenshotHandler.js b/lib/screenshotHandler.js index c175fd1..3aab753 100644 --- a/lib/screenshotHandler.js +++ b/lib/screenshotHandler.js @@ -1,19 +1,90 @@ -var debug = require('debug')('ylt:screenshotHandler'); -var sharp = require('sharp'); +var debug = require('debug')('ylt:screenshotHandlerAgent'); +var Jimp = require('jimp'); var Q = require('q'); var fs = require('fs'); var path = require('path'); -// Disable sharp cache to reduce the "disk is full" error on Amazon Lambda -sharp.cache(false); var screenshotHandler = function() { - this.findAndOptimizeScreenshot = async function(tmpScreenshotPath, width) { - return sharp(tmpScreenshotPath) - .resize({width: 600}) - .jpeg({quality: 85}) - .toBuffer(); + this.findAndOptimizeScreenshot = function(tmpScreenshotPath, width) { + var that = this; + + debug('Starting screenshot transformation'); + + return this.openImage(tmpScreenshotPath) + + .then(function(image) { + that.deleteTmpFile(tmpScreenshotPath); + return that.resizeImage(image, width); + }) + + .then(this.toBuffer); + }; + + + this.openImage = function(imagePath) { + var deferred = Q.defer(); + + Jimp.read(imagePath, function(err, image){ + if (err) { + debug('Could not open imagePath %s', imagePath); + debug(err); + + deferred.reject(err); + } else { + debug('Image correctly open'); + deferred.resolve(image); + } + }); + + return deferred.promise; + }; + + + this.resizeImage = function(image, newWidth) { + var deferred = Q.defer(); + + var currentWidth = image.bitmap.width; + + if (currentWidth > 0) { + var ratio = newWidth / currentWidth; + + image.scale(ratio, function(err, image){ + if (err) { + debug('Could not resize image'); + debug(err); + + deferred.reject(err); + } else { + debug('Image correctly resized'); + deferred.resolve(image); + } + }); + } else { + deferred.reject('Could not resize an empty image'); + } + + return deferred.promise; + }; + + + this.toBuffer = function(image) { + var deferred = Q.defer(); + + image.quality(85).getBuffer(Jimp.MIME_JPEG, function(err, buffer){ + if (err) { + debug('Could not save image to buffer'); + debug(err); + + deferred.reject(err); + } else { + debug('Image correctly transformed to buffer'); + deferred.resolve(buffer); + } + }); + + return deferred.promise; }; diff --git a/lib/tools/redownload/contentTypeChecker.js b/lib/tools/redownload/contentTypeChecker.js index a37c7d5..ea06d88 100644 --- a/lib/tools/redownload/contentTypeChecker.js +++ b/lib/tools/redownload/contentTypeChecker.js @@ -1,12 +1,20 @@ -var debug = require('debug')('ylt:contentTypeChecker'); -var Q = require('q'); -var FileType = require('file-type'); -var isSvg = require('is-svg'); -var isJson = require('is-json'); +var debug = require('debug')('ylt:contentTypeChecker'); +var Q = require('q'); +var isJpg = require('is-jpg'); +var isPng = require('is-png'); +var isSvg = require('is-svg'); +var isGif = require('is-gif'); +var isWebp = require('is-webp'); +var isWoff = require('is-woff'); +var isWoff2 = require('is-woff2'); +var isOtf = require('is-otf'); +var isTtf = require('is-ttf'); +var isEot = require('is-eot'); +var isJson = require('is-json'); var ContentTypeChecker = function() { - async function checkContentType(entry) { + function checkContentType(entry) { var deferred = Q.defer(); // Setting isSomething values: @@ -47,12 +55,12 @@ var ContentTypeChecker = function() { var foundType; try { - foundType = await findContentType(entry.weightCheck.bodyBuffer); + foundType = findContentType(entry.weightCheck.bodyBuffer); // If it's an image or a font, then rewrite. if (foundType !== null && (foundType.type === 'image' || foundType.type === 'webfont' || foundType.type === 'json')) { if (foundType.type !== entry.type) { - debug('Content type %s is wrong for %s. It should be %s.', entry.type, entry.url, foundType.type); + debug('Content type %s is wrong for %s. It should be %s.', entry.type, entry.ulr, foundType.type); } rewriteContentType(entry, foundType); } @@ -68,21 +76,52 @@ var ContentTypeChecker = function() { return deferred.promise; } - async function findContentType(bodyBuffer) { + function findContentType(bodyBuffer) { var bodyStr = bodyBuffer.toString(); + if (isJpg(bodyBuffer)) { + return contentTypes.jpeg; + } + + if (isPng(bodyBuffer)) { + return contentTypes.png; + } + // https://github.com/sindresorhus/is-svg/issues/7 if (/ 0 && gainIsEnough(fileSize, webpFileSize)) { - entry.weightCheck.webpSize = webpFileSize; - debug('WebP size is %d bytes smaller (-%d%)', fileSize - webpFileSize, Math.round((fileSize - webpFileSize) * 100 / fileSize)); - } - - } else { - debug('Convertion to WebP didn\'t work'); - } - - } catch(err) { - debug('Error while converting to WebP, ignoring'); - } - } - - if (!animated && (isJPEG(entry) || isPNG(entry) || isWebP(entry))) { - debug('File is %s and is not animated, let\'s try to convert it to AVIF', entry.contentType); - - try { - - const avifFile = await convertToAvif(entry.weightCheck.bodyBuffer); - - if (avifFile) { - var avifFileSize = avifFile.length; - - debug('AVIF transformation complete for %s', entry.url); - debug('AVIF size is %d bytes', avifFileSize); - - if (avifFile.length > 0 && gainIsEnough(fileSize, avifFileSize)) { - entry.weightCheck.avifSize = avifFileSize; - debug('AVIF size is %d bytes smaller (-%d%)', fileSize - avifFileSize, Math.round((fileSize - avifFileSize) * 100 / fileSize)); - } - - } else { - debug('Convertion to AVIF didn\'t work'); - } - - } catch(err) { - debug('Error while converting to AVIF, ignoring'); - } - } - - return entry; - } - - async function convertToWebp(bodyBuffer, isAnimated) { - return sharp(bodyBuffer, {animated: isAnimated}) - .webp({quality: WEBP_QUALITY, alphaQuality: WEBP_QUALITY}) - .toBuffer(); - } - - async function convertToAvif(bodyBuffer) { - return sharp(bodyBuffer) - .webp({quality: AVIF_QUALITY}) - .toBuffer(); - } - - // The gain is estimated of enough value if it's over 2KB or over 20%, - // but it's ignored if is below 100 bytes - function gainIsEnough(oldWeight, newWeight) { - var gain = oldWeight - newWeight; - var ratio = gain / oldWeight; - return (gain > 2048 || (ratio > 0.2 && gain > 100)); - } - - function isJPEG(entry) { - return entry.isImage && entry.contentType === 'image/jpeg'; - } - - function isPNG(entry) { - return entry.isImage && entry.contentType === 'image/png'; - } - - function isWebP(entry) { - return entry.isImage && entry.contentType === 'image/webp'; - } - - function entryTypeCanBeReformated(entry) { - return isJPEG(entry) || isPNG(entry) || isWebP(entry); - } - - async function isAnimated(entry) { - if (isWebP(entry)) { - const metadata = await sharp(entry.weightCheck.bodyBuffer).metadata(); - return metadata.pages > 1; - } - return false; - } - - return { - reformatImage: reformatImage, - convertToWebp: convertToWebp, - convertToAvif: convertToAvif, - gainIsEnough: gainIsEnough, - entryTypeCanBeReformated: entryTypeCanBeReformated, - isAnimated: isAnimated - }; -}; - -module.exports = new ImageOptimizer(); \ No newline at end of file diff --git a/lib/tools/redownload/redownload.js b/lib/tools/redownload/redownload.js index c567729..803b0dd 100644 --- a/lib/tools/redownload/redownload.js +++ b/lib/tools/redownload/redownload.js @@ -15,7 +15,6 @@ var request = require('request'); var md5 = require('md5'); var imageOptimizer = require('./imageOptimizer'); -var imageReformater = require('./imageReformater'); var fileMinifier = require('./fileMinifier'); var gzipCompressor = require('./gzipCompressor'); var brotliCompressor = require('./brotliCompressor'); @@ -79,10 +78,6 @@ var Redownload = function() { .then(imageOptimizer.optimizeImage) - .then(function(entry) { - return Q(imageReformater.reformatImage(entry)); - }) - .then(imageDimensions.getDimensions) .then(fileMinifier.minifyFile) @@ -96,7 +91,7 @@ var Redownload = function() { }) .then(function(newEntry) { - debug('File %s - Redownloaded, optimized, reformated, minified, compressed, analyzed: done', entry.url); + debug('File %s - Redownloaded, optimized, minified, compressed, analyzed: done', entry.url); // For the progress bar doneCount ++; @@ -159,10 +154,6 @@ var Redownload = function() { offenders.imageOptimization = listImagesNotOptimized(results); metrics.imageOptimization = offenders.imageOptimization.totalGain; - // Old image formats - offenders.oldImageFormats = listImagesWithOldFormats(results); - metrics.oldImageFormats = offenders.oldImageFormats.totalGain; - // Image width offenders.imagesTooLarge = listImagesTooLarge(results, data.params.options.device); metrics.imagesTooLarge = offenders.imagesTooLarge.length; @@ -407,67 +398,6 @@ var Redownload = function() { return results; } - function listImagesWithOldFormats(requests) { - var results = { - totalGain: 0, - images: [] - }; - - requests.forEach(function(req) { - - if (req.weightCheck.bodySize > 0 && - imageReformater.entryTypeCanBeReformated(req) && - (req.weightCheck.webpSize > 0 || req.weightCheck.avifSize > 0)) { - - var image = { - url: req.url, - originalWeigth: req.weightCheck.bodySize, - }; - - switch (req.contentType) { - case 'image/jpeg': - image.originalFormat = 'JPEG'; - break; - case 'image/png': - image.originalFormat = 'PNG'; - break; - case 'image/gif': - image.originalFormat = 'GIF'; - break; - case 'image/webp': - image.originalFormat = 'WebP'; - break; - case 'image/avif': - image.originalFormat = 'AVIF'; - break; - } - - if (req.weightCheck.webpSize) { - image.webpSize = req.weightCheck.webpSize; - image.webpGain = req.weightCheck.bodySize - req.weightCheck.webpSize; - - image.bestFormat = 'WebP'; - image.maxGain = image.webpGain; - } - - if (req.weightCheck.avifSize) { - image.avifSize = req.weightCheck.avifSize; - image.avifGain = req.weightCheck.bodySize - req.weightCheck.avifSize; - - if (!req.weightCheck.webpSize || req.weightCheck.webpSize > req.weightCheck.avifSize) { - image.bestFormat = 'AVIF'; - image.maxGain = image.avifGain; - } - } - - results.totalGain += image.maxGain; - results.images.push(image); - } - }); - - return results; - } - function listImagesTooLarge(requests, device) { var results = []; @@ -889,9 +819,9 @@ var Redownload = function() { debug('Downloading %s', entry.url); - // Always add compression and webp/avif headers before sending, in case the server listens to them + // Always add compression and webp headers before sending, in case the server listens to them var reqHeaders = []; - reqHeaders['Accept'] = '*/*,image/webp,image/avif'; + reqHeaders['Accept'] = '*/*,image/webp'; reqHeaders['Accept-Encoding'] = 'gzip, deflate, br'; reqHeaders['Connection'] = 'keep-alive'; reqHeaders['User-Agent'] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.182 Safari/537.36'; diff --git a/package.json b/package.json index ef327b9..07e5917 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,6 @@ "css-mq-parser": "0.0.3", "debug": "4.3.4", "easyxml": "2.0.1", - "file-type": "16.5.3", "fontkit": "2.0.2", "html-minifier": "4.0.0", "image-size": "1.0.2", @@ -35,15 +34,24 @@ "imagemin-jpegtran": "7.0.0", "imagemin-optipng": "8.0.0", "imagemin-svgo": "9.0.0", + "is-eot": "1.0.0", + "is-gif": "3.0.0", + "is-jpg": "2.0.0", "is-json": "2.0.1", + "is-otf": "0.1.2", + "is-png": "1.1.0", "is-svg": "3.0.0", + "is-ttf": "0.2.2", + "is-webp": "1.0.1", + "is-woff": "1.0.3", + "is-woff2": "1.0.0", + "jimp": "0.22.8", "md5": "2.3.0", "meow": "5.0.0", "parse-color": "1.0.0", "phantomas": "2.8.0", "q": "1.5.1", "request": "2.88.2", - "sharp": "0.32.3", "ttf2woff2": "5.0.0", "uglify-js": "3.17.4", "woff-tools": "0.1.0" diff --git a/test/core/contentTypeCheckerTest.js b/test/core/contentTypeCheckerTest.js index 47311d5..ac67cc4 100644 --- a/test/core/contentTypeCheckerTest.js +++ b/test/core/contentTypeCheckerTest.js @@ -10,11 +10,11 @@ describe('contentTypeChecker', function() { var svgImageContent = fs.readFileSync(path.resolve(__dirname, '../www/svg-image.svg')); var cssFileContent = fs.readFileSync(path.resolve(__dirname, '../www/unminified-stylesheet.css')); - it('detect the right content type', async function() { - (await contentTypeChecker.findContentType(jpgImageContent)).mimes.should.deep.equal(['image/jpeg']); - (await contentTypeChecker.findContentType(pngImageContent)).mimes.should.deep.equal(['image/png']); - (await contentTypeChecker.findContentType(svgImageContent)).mimes.should.deep.equal(['image/svg+xml']); - should.equal(await contentTypeChecker.findContentType(cssFileContent), null); + it('detect the right content type', function() { + contentTypeChecker.findContentType(jpgImageContent).mimes.should.deep.equal(['image/jpeg']); + contentTypeChecker.findContentType(pngImageContent).mimes.should.deep.equal(['image/png']); + contentTypeChecker.findContentType(svgImageContent).mimes.should.deep.equal(['image/svg+xml']); + should.equal(contentTypeChecker.findContentType(cssFileContent), null); }); }); diff --git a/test/core/imageReformaterTest.js b/test/core/imageReformaterTest.js deleted file mode 100644 index 07d6541..0000000 --- a/test/core/imageReformaterTest.js +++ /dev/null @@ -1,122 +0,0 @@ -var should = require('chai').should(); -var imageReformater = require('../../lib/tools/redownload/imageReformater'); -var fs = require('fs'); -var path = require('path'); - -describe('imageReformater', function() { - - it('should convert a JPEG image to WebP and AVIF', async function() { - var fileContent = fs.readFileSync(path.resolve(__dirname, '../www/jpeg-image.jpg')); - let entry = { - isImage: true, - type: 'image', - contentType: 'image/jpeg', - weightCheck: { - bodyBuffer: fileContent, - uncompressedSize: fileContent.length - } - }; - - var newEntry = await imageReformater.reformatImage(entry); - - newEntry.weightCheck.should.have.a.property('webpSize'); - newEntry.weightCheck.webpSize.should.be.below(fileContent.length); - - newEntry.weightCheck.should.have.a.property('avifSize'); - newEntry.weightCheck.avifSize.should.be.below(fileContent.length); - }); - - it('should convert a PNG image to WebP and AVIF', async function() { - var fileContent = fs.readFileSync(path.resolve(__dirname, '../www/jpeg-image.jpg')); - let entry = { - isImage: true, - type: 'image', - contentType: 'image/png', - weightCheck: { - bodyBuffer: fileContent, - uncompressedSize: fileContent.length - } - }; - - var newEntry = await imageReformater.reformatImage(entry); - - newEntry.weightCheck.should.have.a.property('webpSize'); - newEntry.weightCheck.webpSize.should.be.below(fileContent.length); - - newEntry.weightCheck.should.have.a.property('avifSize'); - newEntry.weightCheck.avifSize.should.be.below(fileContent.length); - }); - - it('should convert a WebP image to AVIF', async function() { - var fileContent = fs.readFileSync(path.resolve(__dirname, '../www/jpeg-image.jpg')); - let entry = { - isImage: true, - type: 'image', - contentType: 'image/webp', - weightCheck: { - bodyBuffer: fileContent, - uncompressedSize: fileContent.length - } - }; - - var newEntry = await imageReformater.reformatImage(entry); - - newEntry.weightCheck.should.not.have.a.property('webpSize'); - - newEntry.weightCheck.should.have.a.property('avifSize'); - newEntry.weightCheck.avifSize.should.be.below(fileContent.length); - }); - - it('should recognize an animated WebP', async function() { - // Test on an animated image - let fileContent = fs.readFileSync(path.resolve(__dirname, '../www/animated.webp')); - let entry = { - isImage: true, - type: 'image', - contentType: 'image/webp', - weightCheck: { - bodyBuffer: fileContent, - uncompressedSize: fileContent.length - } - }; - - (await imageReformater.isAnimated(entry)).should.equal(true); - - // Test on a not animated image - fileContent = fs.readFileSync(path.resolve(__dirname, '../www/monster.webp')); - entry.weightCheck.bodyBuffer = fileContent; - (await imageReformater.isAnimated(entry)).should.equal(false); - }); - - it('should not convert an animated WebP', async function() { - // Test on an animated image - let fileContent = fs.readFileSync(path.resolve(__dirname, '../www/animated.webp')); - let entry = { - isImage: true, - type: 'image', - contentType: 'image/webp', - weightCheck: { - bodyBuffer: fileContent, - uncompressedSize: fileContent.length - } - }; - - var newEntry = await imageReformater.reformatImage(entry); - - // Test on a not animated image - newEntry.weightCheck.should.not.have.a.property('avifSize'); - }); - - it('should determine if gain is enough', function() { - imageReformater.gainIsEnough(20000, 10000).should.equal(true); - imageReformater.gainIsEnough(2000, 1000).should.equal(true); - imageReformater.gainIsEnough(20000, 21000).should.equal(false); - imageReformater.gainIsEnough(20000, 40000).should.equal(false); - imageReformater.gainIsEnough(20000, 19500).should.equal(false); - imageReformater.gainIsEnough(250, 120).should.equal(true); - imageReformater.gainIsEnough(200, 120).should.equal(false); - imageReformater.gainIsEnough(2000, 1900).should.equal(false); - imageReformater.gainIsEnough(200000, 197000).should.equal(true); - }); - -}); diff --git a/test/core/redownloadTest.js b/test/core/redownloadTest.js index ada3e22..0c02dae 100644 --- a/test/core/redownloadTest.js +++ b/test/core/redownloadTest.js @@ -86,10 +86,6 @@ describe('redownload', function() { data.toolsResults.redownload.offenders.imageOptimization.totalGain.should.be.above(0); data.toolsResults.redownload.offenders.imageOptimization.images.length.should.equal(2); - data.toolsResults.redownload.offenders.should.have.a.property('oldImageFormats'); - data.toolsResults.redownload.offenders.oldImageFormats.totalGain.should.be.above(0); - data.toolsResults.redownload.offenders.oldImageFormats.images.length.should.equal(1); - data.toolsResults.redownload.offenders.should.have.a.property('imagesTooLarge'); data.toolsResults.redownload.offenders.imagesTooLarge.length.should.equal(0); @@ -171,9 +167,18 @@ describe('redownload', function() { redownload.redownloadEntry(entry) .then(function(newEntry) { + newEntry.weightCheck.bodySize.should.equal(4193); newEntry.weightCheck.bodyBuffer.should.deep.equal(fileContent); - done(); + + // Opening the image in jimp to check if the format is good + var Jimp = require('jimp'); + Jimp.read(newEntry.weightCheck.bodyBuffer, function(err, image) { + image.bitmap.width.should.equal(620); + image.bitmap.height.should.equal(104); + done(err); + }); + }) .fail(function(err) { diff --git a/test/www/animated.webp b/test/www/animated.webp deleted file mode 100644 index 7a1d3fe..0000000 Binary files a/test/www/animated.webp and /dev/null differ diff --git a/test/www/monster.webp b/test/www/monster.webp deleted file mode 100644 index e6ec320..0000000 Binary files a/test/www/monster.webp and /dev/null differ