From b201b32b46aa5a5daf5e651bd94eb65d9313fe63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABl=20M=C3=A9tais?= Date: Tue, 10 Jan 2017 13:54:43 +0800 Subject: [PATCH 1/6] Hiddenchars (#221) * Fix bug where an icons font is counted as two unused unicode ranges * Display the number of hidden chars --- front/src/views/rule.html | 5 +++++ lib/metadata/policies.js | 2 +- lib/tools/redownload/redownload.js | 20 ++++++++++++++++---- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/front/src/views/rule.html b/front/src/views/rule.html index f8d33d8..c8c00fb 100644 --- a/front/src/views/rule.html +++ b/front/src/views/rule.html @@ -455,6 +455,11 @@ +
+
Ligatures or hidden chars
+
{{font.ligaturesOrHiddenChars}} glyphs
+
+
diff --git a/lib/metadata/policies.js b/lib/metadata/policies.js index 0bfa76f..d00f6e0 100644 --- a/lib/metadata/policies.js +++ b/lib/metadata/policies.js @@ -1062,7 +1062,7 @@ var policies = { "unusedUnicodeRanges": { "tool": "redownload", "label": "Unused Unicode ranges", - "message": "

This metric counts the number of unused Unicode ranges inside each font. For example, one font could include Cyrillic glyphs but none of them are used on the page.

Because of technical limitations, Yellow Lab Tools checks each font against the glyphs of the entire page. As a result, estimated use is >= to reality. For example, if you read that 10 glyphs are \"possibly used\", it means that these 10 glyphs are used on the page but nothing guaranties that they are displayed using this font.

Tools such as Font Squirrel can remove some unicode ranges from a font.

In the case of an icon font, make sure you only keep the icons that are used on the website and to remove the others. Several tools are able to extract SVG images from a font, then some other tools can generate a font from the SVGs you want to keep.

", + "message": "

This metric counts the number of unused Unicode ranges inside each font. For example, one font could include Cyrillic glyphs but none of them are used on the page.

It also reveals the number of ligatures (letters that are represented differently when close to each other) and hidden chars (glyphs not linked to the unicode system that can't be displayed on the web).

Because of technical limitations, Yellow Lab Tools checks each font against the glyphs of the entire page. As a result, estimated use is >= to reality. For example, if you read that 10 glyphs are \"possibly used\", it means that these 10 glyphs are used on the page but nothing guaranties that they are displayed using this font.

Tools such as Font Squirrel can remove some unicode ranges from a font.

In the case of an icon font, make sure you only keep the icons that are used on the website and to remove the others. Several tools are able to extract SVG images from a font, then some other tools can generate a font from the SVGs you want to keep.

", "isOkThreshold": 0, "isBadThreshold": 8, "isAbnormalThreshold": 12, diff --git a/lib/tools/redownload/redownload.js b/lib/tools/redownload/redownload.js index c79c71c..0e34fff 100644 --- a/lib/tools/redownload/redownload.js +++ b/lib/tools/redownload/redownload.js @@ -500,6 +500,8 @@ var Redownload = function() { var ranges = []; var others = null; var rangeNames = Object.keys(req.fontMetrics.unicodeRanges); + var unicodePointsCount = 0; + var unusedRangesInFont = 0; rangeNames.forEach(function(rangeName) { var range = req.fontMetrics.unicodeRanges[rangeName]; @@ -508,18 +510,20 @@ var Redownload = function() { if (rangeName === 'Others') { if (range.numGlyphsInCommonWithPageContent === 0 && range.charset.length > 50) { range.underused = true; - unusedUnicodeRanges ++; + unusedRangesInFont ++; } + unicodePointsCount += range.charset.length; others = range; } else if (range.charset.length > 0) { // Now lets detect if the current Unicode range is unused. // Reminder: range.coverage = glyphs declared in this range, divided by the range size if (range.coverage > 0.25 && range.numGlyphsInCommonWithPageContent === 0) { range.underused = true; - unusedUnicodeRanges ++; + unusedRangesInFont ++; } + unicodePointsCount += range.charset.length; ranges.push(range); } }); @@ -539,7 +543,7 @@ var Redownload = function() { // And if less than 5% of the icons are used, let's report it as underused if (others && others.numGlyphsInCommonWithPageContent / others.charset.length <= 0.05) { - unusedUnicodeRanges ++; + unusedRangesInFont = 1; } // Not an icons font @@ -549,13 +553,21 @@ var Redownload = function() { ranges.push(others); } + var ligaturesOrHiddenChars = req.fontMetrics.numGlyphs - unicodePointsCount; + if (ligaturesOrHiddenChars > 25) { + unusedUnicodeRanges ++; + } + list.push({ url: req.url, weight: req.weightCheck.bodySize, isIconFont: false, - unicodeRanges: ranges + unicodeRanges: ranges, + ligaturesOrHiddenChars: ligaturesOrHiddenChars }); } + + unusedUnicodeRanges += unusedRangesInFont; } }); From 3555d01b689c75aa5e1a5039668a73325e89c616 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABl=20M=C3=A9tais?= Date: Sat, 27 May 2017 18:18:09 +0100 Subject: [PATCH 2/6] Add file names to duplicated css selectors offenders (#233) --- front/src/views/rule.html | 8 ++++---- lib/metadata/policies.js | 30 +++++++++++++++--------------- package.json | 2 +- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/front/src/views/rule.html b/front/src/views/rule.html index c8c00fb..f011a2a 100644 --- a/front/src/views/rule.html +++ b/front/src/views/rule.html @@ -117,10 +117,6 @@ -
- {{offender.rule}} (x{{offender.occurrences}}) -
-
{{offender.property}} {{offender.message}}
@@ -188,6 +184,10 @@ @ {{offender.line}}:{{offender.column}}
+
+ {{offender.rule}} (x{{offender.occurrences}}) +
+
Property {{offender.property}} duplicated in {{offender.rule}} { } @ {{offender.line}}:{{offender.column}} diff --git a/lib/metadata/policies.js b/lib/metadata/policies.js index d00f6e0..0a1df51 100644 --- a/lib/metadata/policies.js +++ b/lib/metadata/policies.js @@ -596,24 +596,24 @@ var policies = { "isAbnormalThreshold": 100, "hasOffenders": true, "offendersTransformFn": function(offenders) { - return { - count: offenders.length, - list: offenders.map(function(offender) { - var parts = /^(.*) \((\d+) times\)$/.exec(offender); - - if (!parts) { - debug('cssDuplicatedSelectors offenders transform function error with "%s"', offender); - return { - parseError: offender - }; - } + var parsedOffenders = offenders.map(function(offender) { + var parts = /^(.*) \((\d+) times\) ?<(.*)>$/.exec(offender); + if (!parts) { + debug('cssDuplicatedSelectors offenders transform function error with "%s"', offender); return { - rule: parts[1], - occurrences: parseInt(parts[2], 10) + parseError: offender }; - }) - }; + } + + return { + rule: parts[1], + occurrences: parseInt(parts[2], 10), + file: parts[3] + }; + }); + + return offendersHelpers.orderByFile(parsedOffenders); } }, "cssDuplicatedProperties": { diff --git a/package.json b/package.json index 23b089b..b8da64a 100644 --- a/package.json +++ b/package.json @@ -60,7 +60,7 @@ "meow": "3.7.0", "minimize": "2.0.0", "parse-color": "1.0.0", - "phantomas": "1.18.0", + "phantomas": "1.19.0", "ps-node": "0.1.4", "q": "1.4.1", "request": "2.79.0", From f59c8da000fc2d393f8b5495cc919f068349d31d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABl=20M=C3=A9tais?= Date: Sat, 27 May 2017 23:14:20 +0100 Subject: [PATCH 3/6] Ignore unused fonts (#234) --- lib/tools/redownload/fontAnalyzer.js | 11 ++++++++++- lib/tools/redownload/redownload.js | 8 ++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/lib/tools/redownload/fontAnalyzer.js b/lib/tools/redownload/fontAnalyzer.js index d83d0a8..0fab2f5 100644 --- a/lib/tools/redownload/fontAnalyzer.js +++ b/lib/tools/redownload/fontAnalyzer.js @@ -57,8 +57,17 @@ var FontAnalyzer = function() { var endTime = Date.now(); debug('Font analysis took %dms', endTime - startTime); - deferred.resolve(result); + // Mark fonts that are not used on the page (#224) + var fontIsUsed = false; + for (var range in result.unicodeRanges) { + if (result.unicodeRanges[range].numGlyphsInCommonWithPageContent > 0) { + fontIsUsed = true; + break; + } + } + result.isUsed = fontIsUsed; + deferred.resolve(result); } catch(error) { deferred.reject(error); } diff --git a/lib/tools/redownload/redownload.js b/lib/tools/redownload/redownload.js index 0e34fff..824d6ed 100644 --- a/lib/tools/redownload/redownload.js +++ b/lib/tools/redownload/redownload.js @@ -103,6 +103,14 @@ var Redownload = function() { var metrics = {}; var offenders = {}; + // Remove unused fonts that a normal browser would not download (fix #224) + results = results.filter(function(result) { + if (result && result.fontMetrics) { + return result.fontMetrics.isUsed !== false; + } + return true; + }); + // Count requests offenders.totalRequests = listRequestsByType(results); metrics.totalRequests = offenders.totalRequests.total; From 40b3fa91ac21c5e20636f56c2cb7ded0d0e3ccdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABl=20M=C3=A9tais?= Date: Sat, 27 May 2017 23:17:24 +0100 Subject: [PATCH 4/6] v1.12.2 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index b8da64a..403b787 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "yellowlabtools", - "version": "1.12.1", + "version": "1.12.2", "description": "Online tool to audit a webpage for performance and front-end quality issues", "license": "GPL-2.0", "author": { From a20c5677b4eab102a29876fea5f4d429d37b812e Mon Sep 17 00:00:00 2001 From: Catherine Snow Date: Thu, 16 Nov 2017 11:48:49 -0500 Subject: [PATCH 5/6] Fix typos --- front/src/js/directives/offendersDirectives.js | 4 ++-- lib/metadata/policies.js | 2 +- lib/tools/redownload/fileMinifier.js | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/front/src/js/directives/offendersDirectives.js b/front/src/js/directives/offendersDirectives.js index 609871b..f9d747e 100644 --- a/front/src/js/directives/offendersDirectives.js +++ b/front/src/js/directives/offendersDirectives.js @@ -751,7 +751,7 @@ } function onDetailsClick(row) { - // Close if it's alreay open + // Close if it's already open if (row.classList.contains('showDetails')) { closeDetails(row); return; @@ -924,4 +924,4 @@ }; }); -})(); \ No newline at end of file +})(); diff --git a/lib/metadata/policies.js b/lib/metadata/policies.js index 0a1df51..3a20d32 100644 --- a/lib/metadata/policies.js +++ b/lib/metadata/policies.js @@ -893,7 +893,7 @@ var policies = { "totalWeight": { "tool": "redownload", "label": "Total weight", - "message": "

The weight is of course very important if you want the page to load fast. Try to stay under 1MB, which is alreay very long to download over a slow connection.

Please note that Yellow Lab Tools' engine (PhantomJS) is not compatible with image srcset (unless you use a polyfill). This can lead to incorrect page weight.

", + "message": "

The weight is of course very important if you want the page to load fast. Try to stay under 1MB, which is already very long to download over a slow connection.

Please note that Yellow Lab Tools' engine (PhantomJS) is not compatible with image srcset (unless you use a polyfill). This can lead to incorrect page weight.

", "isOkThreshold": 716800, "isBadThreshold": 2097152, "isAbnormalThreshold": 3145728, diff --git a/lib/tools/redownload/fileMinifier.js b/lib/tools/redownload/fileMinifier.js index fc9ea1a..4f9371a 100644 --- a/lib/tools/redownload/fileMinifier.js +++ b/lib/tools/redownload/fileMinifier.js @@ -296,7 +296,7 @@ var FileMinifier = function() { return deferred.promise; } - // Avoid loosing time trying to compress some JS libraries known as already compressed + // Avoid losing time trying to compress some JS libraries known as already compressed function isKnownAsMinified(url) { var result = false; @@ -322,7 +322,7 @@ var FileMinifier = function() { return result; } - // Avoid loosing some trying to compress JS files if they alreay look minified + // Avoid losing some trying to compress JS files if they already look minified // by counting the number of lines compared to the total size. // Less than 2KB per line is suspicious function looksAlreadyMinified(code) { @@ -351,4 +351,4 @@ var FileMinifier = function() { }; }; -module.exports = new FileMinifier(); \ No newline at end of file +module.exports = new FileMinifier(); From 3fc0d9418fc2cca975ae20f3957cee5729de258d Mon Sep 17 00:00:00 2001 From: Paul Hooijenga Date: Wed, 13 Dec 2017 15:54:36 +0100 Subject: [PATCH 6/6] Use jimp instead of lwip for image processing Jimp does not install on recent versions of NodeJS because of changes in the bundled version of zlib. Jimp is a pure JS replacement. --- lib/screenshotHandler.js | 16 ++++++++-------- package.json | 2 +- test/api/screenshotHandlerTest.js | 32 +++++++++++++++---------------- test/core/redownloadTest.js | 12 ++++++------ 4 files changed, 31 insertions(+), 31 deletions(-) diff --git a/lib/screenshotHandler.js b/lib/screenshotHandler.js index b803ed9..b960672 100644 --- a/lib/screenshotHandler.js +++ b/lib/screenshotHandler.js @@ -1,5 +1,5 @@ var debug = require('debug')('ylt:screenshotHandler'); -var lwip = require('lwip'); +var Jimp = require('jimp'); var tmp = require('temporary'); var Q = require('q'); var fs = require('fs'); @@ -57,7 +57,7 @@ var screenshotHandler = function() { this.openImage = function(imagePath) { var deferred = Q.defer(); - lwip.open(imagePath, function(err, image){ + Jimp.read(imagePath, function(err, image){ if (err) { debug('Could not open imagePath %s', imagePath); debug(err); @@ -76,7 +76,7 @@ var screenshotHandler = function() { this.resizeImage = function(image, newWidth) { var deferred = Q.defer(); - var currentWidth = image.width(); + var currentWidth = image.bitmap.width; var ratio = newWidth / currentWidth; image.scale(ratio, function(err, image){ @@ -101,7 +101,7 @@ var screenshotHandler = function() { var deferred = Q.defer(); // Create a canvas with the same dimensions as your image: - lwip.create(image.width(), image.height(), 'white', function(err, canvas){ + new Jimp(image.bitmap.width, image.bitmap.height, 0xFFFFFF, function(err, canvas){ if (err) { debug('Could not create a white canvas'); debug(err); @@ -109,7 +109,7 @@ var screenshotHandler = function() { deferred.reject(err); } else { // Paste original image on top of the canvas - canvas.paste(0, 0, image, function(err, image){ + canvas.composite(image, 0, 0, function(err, image){ if (err) { debug('Could not paste image on the white canvas'); debug(err); @@ -131,7 +131,7 @@ var screenshotHandler = function() { this.toBuffer = function(image) { var deferred = Q.defer(); - image.toBuffer('jpg', {quality: 90}, function(err, buffer){ + image.quality(90).getBuffer(Jimp.MIME_JPEG, function(err, buffer){ if (err) { debug('Could not save image to buffer'); debug(err); @@ -143,7 +143,7 @@ var screenshotHandler = function() { } }); - return deferred.promise; + return deferred.promise; }; @@ -167,4 +167,4 @@ var screenshotHandler = function() { }; }; -module.exports = new screenshotHandler(); \ No newline at end of file +module.exports = new screenshotHandler(); diff --git a/package.json b/package.json index 52bb7ae..ef02123 100644 --- a/package.json +++ b/package.json @@ -55,7 +55,7 @@ "is-ttf": "0.2.2", "is-woff": "1.0.3", "is-woff2": "1.0.0", - "lwip": "0.0.9", + "jimp": "0.2.28", "md5": "2.2.1", "meow": "3.7.0", "minimize": "2.0.0", diff --git a/test/api/screenshotHandlerTest.js b/test/api/screenshotHandlerTest.js index 675e3b7..267a44e 100644 --- a/test/api/screenshotHandlerTest.js +++ b/test/api/screenshotHandlerTest.js @@ -7,17 +7,17 @@ var path = require('path'); describe('screenshotHandler', function() { var imagePath = path.join(__dirname, '../fixtures/logo-large.png'); - var screenshot, lwipImage; + var screenshot, jimpImage; - - it('should open an image and return an lwip object', function(done) { + + it('should open an image and return an jimp object', function(done) { ScreenshotHandler.openImage(imagePath) .then(function(image) { - lwipImage = image; + jimpImage = image; - lwipImage.should.be.an('object'); - lwipImage.width().should.equal(620); - lwipImage.height().should.equal(104); + jimpImage.should.be.an('object'); + jimpImage.bitmap.width.should.equal(620); + jimpImage.bitmap.height.should.equal(104); done(); }) @@ -26,14 +26,14 @@ describe('screenshotHandler', function() { }); }); - - it('should resize an lwip image', function(done) { - ScreenshotHandler.resizeImage(lwipImage, 310) + + it('should resize an jimp image', function(done) { + ScreenshotHandler.resizeImage(jimpImage, 310) .then(function(image) { - lwipImage = image; + jimpImage = image; - lwipImage.width().should.equal(310); - lwipImage.height().should.equal(52); + jimpImage.bitmap.width.should.equal(310); + jimpImage.bitmap.height.should.equal(52); done(); }) @@ -43,8 +43,8 @@ describe('screenshotHandler', function() { }); - it('should transform a lwip image into a buffer', function(done) { - ScreenshotHandler.toBuffer(lwipImage) + it('should transform a jimp image into a buffer', function(done) { + ScreenshotHandler.toBuffer(jimpImage) .then(function(buffer) { buffer.should.be.an.instanceof(Buffer); done(); @@ -125,4 +125,4 @@ describe('screenshotHandler', function() { done(err); }); }); -}); \ No newline at end of file +}); diff --git a/test/core/redownloadTest.js b/test/core/redownloadTest.js index 0b65ee1..f4890ee 100644 --- a/test/core/redownloadTest.js +++ b/test/core/redownloadTest.js @@ -228,14 +228,14 @@ describe('redownload', function() { newEntry.weightCheck.bodySize.should.equal(4193); newEntry.weightCheck.bodyBuffer.should.deep.equal(fileContent); - // Opening the image in lwip to check if the format is good - var lwip = require('lwip'); - lwip.open(newEntry.weightCheck.bodyBuffer, 'png', function(err, image) { - image.width().should.equal(620); - image.height().should.equal(104); + // Opening the image in jimp to check if the format is good + var Jimp = require('jimp'); + Jimp.read(newEntry.weightCheck.bodyBuffer, function(err, image) { + image.bitmap.width.should.equal(620); + image.bitmap.height.should.equal(104); done(err); }); - + }) .fail(function(err) {