diff --git a/Gruntfile.js b/Gruntfile.js index b7c5fea..7bf74da 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -143,7 +143,7 @@ module.exports = function(grunt) { options: { reporter: 'spec', }, - src: ['test/core/offendersHelpersTest.js'] + src: ['test/core/phantomasWrapperTest.js'] }, coverage: { options: { diff --git a/lib/runner.js b/lib/runner.js index df8423f..43ead1e 100644 --- a/lib/runner.js +++ b/lib/runner.js @@ -3,6 +3,7 @@ var debug = require('debug')('ylt:runner'); var phantomasWrapper = require('./tools/phantomas/phantomasWrapper'); var jsExecutionTransformer = require('./tools/jsExecutionTransformer'); +var weightChecker = require('./tools/weightChecker'); var rulesChecker = require('./rulesChecker'); var scoreCalculator = require('./scoreCalculator'); @@ -25,8 +26,10 @@ var Runner = function(params) { // Treat the JS Execution Tree from offenders data = jsExecutionTransformer.transform(data); - // Other tools go here + // Redownload every file + return weightChecker.recheckAllFiles(data); + }).then(function(data) { // Rules checker var policies = require('./metadata/policies'); diff --git a/lib/tools/phantomas/custom_modules/modules/requestsList/requestsList.js b/lib/tools/phantomas/custom_modules/modules/requestsList/requestsList.js new file mode 100644 index 0000000..a36bfe1 --- /dev/null +++ b/lib/tools/phantomas/custom_modules/modules/requestsList/requestsList.js @@ -0,0 +1,23 @@ +/** + * Retries download on every request to get the real file size + * + */ + +exports.version = '0.1'; + +exports.module = function(phantomas) { + 'use strict'; + + phantomas.setMetric('requestsList'); + + var requests = []; + + phantomas.on('recv', function(entry, res) { + requests.push(entry); + }); + + phantomas.on('report', function() { + phantomas.setMetric('requestsList', true, true); + phantomas.addOffender('requestsList', JSON.stringify(requests)); + }); +}; diff --git a/lib/tools/phantomas/phantomasWrapper.js b/lib/tools/phantomas/phantomasWrapper.js index 1e2a74a..30a3cad 100644 --- a/lib/tools/phantomas/phantomasWrapper.js +++ b/lib/tools/phantomas/phantomasWrapper.js @@ -67,7 +67,7 @@ var PhantomasWrapper = function() { if (value === true) { optionsString += ' ' + '--' + opt; - } else if (value === false || value === null) { + } else if (value === false || value === null || value === undefined) { // Nothing } else { optionsString += ' ' + '--' + opt + '=' + value; @@ -122,7 +122,7 @@ var PhantomasWrapper = function() { } - debug('Returning from Phantomas'); + debug('Returning from Phantomas with error %s', err); // Adding some YellowLabTools errors here if (json && json.metrics && (!json.metrics.javascriptExecutionTree || !json.offenders.javascriptExecutionTree)) { diff --git a/lib/tools/weightChecker.js b/lib/tools/weightChecker.js new file mode 100644 index 0000000..a0d5cea --- /dev/null +++ b/lib/tools/weightChecker.js @@ -0,0 +1,279 @@ +/* + * Redownloading every files after Phantomas has finished + * Checks weight and every kind of compression + * + */ + + +var debug = require('debug')('ylt:weightChecker'); + +var request = require('request'); +var http = require('http'); +var async = require('async'); +var Q = require('Q'); +var zlib = require('zlib'); + +var WeightChecker = function() { + + var MAX_PARALLEL_DOWNLOADS = 10; + var REQUEST_TIMEOUT = 10000; // 10 seconds + + function recheckAllFiles(data) { + var deferred = Q.defer(); + + var requestsList = JSON.parse(data.toolsResults.phantomas.offenders.requestsList); + delete data.toolsResults.phantomas.metrics.requestsList; + delete data.toolsResults.phantomas.offenders.requestsList; + + // Transform every request into a download function with a callback when done + var redownloadList = requestsList.map(function(entry) { + return function(callback) { + redownloadEntry(entry, callback); + }; + }); + + // Lanch all redownload functions and wait for completion + async.parallelLimit(redownloadList, MAX_PARALLEL_DOWNLOADS, function(err, results) { + if (err) { + debug(err); + deferred.reject(err); + } else { + debug('All files checked'); + + var metrics = {}; + var offenders = {}; + + + // Total weight + offenders.totalWeight = listRequestWeight(results); + metrics.totalWeight = offenders.totalWeight.totalWeight; + + + data.toolsResults.weightChecker = { + metrics: metrics, + offenders: offenders + }; + + deferred.resolve(data); + } + }); + + return deferred.promise; + } + + + function listRequestWeight(requests) { + var results = { + totalWeight: 0, + byType: { + html: { + totalWeight: 0, + requests: [] + }, + css: { + totalWeight: 0, + requests: [] + }, + js: { + totalWeight: 0, + requests: [] + }, + json: { + totalWeight: 0, + requests: [] + }, + image: { + totalWeight: 0, + requests: [] + }, + video: { + totalWeight: 0, + requests: [] + }, + webfont: { + totalWeight: 0, + requests: [] + }, + other: { + totalWeight: 0, + requests: [] + } + } + }; + + requests.forEach(function(req) { + var weight = (typeof req.weightCheck.bodySize === 'number') ? req.weightCheck.bodySize + req.weightCheck.headersSize : req.contentLength; + var type = req.type || 'other'; + + results.totalWeight += weight; + results.byType[type].totalWeight += weight; + + results.byType[type].requests.push({ + url: req.url, + weight: weight + }); + }); + + return results; + } + + + function redownloadEntry(entry, callback) { + + function onError(message) { + debug('Could not download %s Error: %s', entry.url, message); + entry.weightCheck = { + message: message + }; + setImmediate(function() { + callback(null, entry); + }); + } + + if (entry.method !== 'GET') { + onError('only downloading GET'); + return; + } + + if (entry.status !== 200) { + onError('only downloading requests with status code 200'); + return; + } + + + debug('Downloading %s', entry.url); + + // Always add a gzip header before sending, in case the server listens to it + var reqHeaders = entry.requestHeaders; + reqHeaders['Accept-Encoding'] = 'gzip, deflate'; + + var requestOptions = { + method: entry.method, + url: entry.url, + headers: reqHeaders, + timeout: REQUEST_TIMEOUT + }; + + download(requestOptions, function(error, result) { + if (error) { + if (error.code === 'ETIMEDOUT') { + onError('timeout after ' + REQUEST_TIMEOUT + 'ms'); + } else { + onError('error while downloading: ' + error.code); + } + return; + } + + debug('%s downloaded correctly', entry.url); + + entry.weightCheck = result; + callback(null, entry); + }); + } + + // Inspired by https://github.com/cvan/fastHAR-api/blob/10cec585/app.js + function download(requestOptions, callback) { + + var statusCode; + + request(requestOptions) + + .on('response', function(res) { + + // Raw headers were added in NodeJS v0.12 + // (https://github.com/joyent/node/issues/4844), but let's + // reconstruct them for backwards compatibility. + var rawHeaders = ('HTTP/' + res.httpVersion + ' ' + res.statusCode + + ' ' + http.STATUS_CODES[res.statusCode] + '\r\n'); + Object.keys(res.headers).forEach(function(headerKey) { + rawHeaders += headerKey + ': ' + res.headers[headerKey] + '\r\n'; + }); + rawHeaders += '\r\n'; + + var uncompressedSize = 0; // size after uncompression + var bodySize = 0; // bytes size over the wire + var body = ''; // plain text body (after uncompressing gzip/deflate) + var isCompressed = false; + + function tally() { + + if (statusCode !== 200) { + callback({code: statusCode}); + return; + } + + var result = { + body: body, + headersSize: Buffer.byteLength(rawHeaders, 'utf8'), + bodySize: bodySize, + isCompressed: isCompressed, + uncompressedSize: uncompressedSize + }; + + callback(null, result); + } + + switch (res.headers['content-encoding']) { + case 'gzip': + var gzip = zlib.createGunzip(); + + gzip.on('data', function (data) { + body += data; + uncompressedSize += data.length; + }).on('end', function () { + isCompressed = true; + tally(); + }); + + res.on('data', function (data) { + bodySize += data.length; + }).pipe(gzip); + + break; + case 'deflate': + var deflate = zlib.createInflate(); + + deflate.on('data', function (data) { + body += data; + uncompressedSize += data.length; + }).on('end', function () { + isCompressed = true; + tally(); + }); + + res.on('data', function (data) { + bodySize += data.length; + }).pipe(deflate); + + break; + default: + res.on('data', function (data) { + body += data; + uncompressedSize += data.length; + bodySize += data.length; + }).on('end', function () { + tally(); + }); + + break; + } + }) + + .on('response', function(response) { + statusCode = response.statusCode; + }) + + .on('error', function(err) { + callback(err); + }); + } + + return { + recheckAllFiles: recheckAllFiles, + listRequestWeight: listRequestWeight, + redownloadEntry: redownloadEntry, + download: download + }; +}; + +module.exports = new WeightChecker(); \ No newline at end of file diff --git a/test/core/phantomasWrapperTest.js b/test/core/phantomasWrapperTest.js index 3615407..28bafc5 100644 --- a/test/core/phantomasWrapperTest.js +++ b/test/core/phantomasWrapperTest.js @@ -42,10 +42,13 @@ describe('phantomasWrapper', function() { phantomasWrapper.execute({ params: { url: url, - options: {} + options: { + device: 'desktop' + } } }).then(function(data) { + console.log(data); done('Error: unwanted success'); }).fail(function(err) { diff --git a/test/core/weightCheckerTest.js b/test/core/weightCheckerTest.js new file mode 100644 index 0000000..61bf429 --- /dev/null +++ b/test/core/weightCheckerTest.js @@ -0,0 +1,183 @@ +var should = require('chai').should(); +var weightChecker = require('../../lib/tools/weightChecker'); + +describe('weightChecker', function() { + + it('should download a list of files', function(done) { + var requestsList = [ + { + method: 'GET', + url: 'http://localhost:8388/simple-page.html', + requestHeaders: { + 'User-Agent': 'something', + Referer: 'http://www.google.fr/', + Accept: '*/*' + }, + status: 200, + isHTML: true + }, + { + method: 'GET', + url: 'http://localhost:8388/jquery1.8.3.js', + requestHeaders: { + 'User-Agent': 'something', + Referer: 'http://www.google.fr/', + Accept: '*/*' + }, + status: 200, + isJS: true + } + ]; + + var data = { + toolsResults: { + phantomas: { + metrics: { + requestsList: true + }, + offenders: { + requestsList: JSON.stringify(requestsList) + } + } + } + }; + + weightChecker.recheckAllFiles(data) + + .then(function(data) { + data.toolsResults.should.have.a.property('weightChecker'); + data.toolsResults.weightChecker.should.have.a.property('metrics'); + data.toolsResults.weightChecker.should.have.a.property('offenders'); + done(); + }) + + .fail(function(err) { + done(err); + }); + }); + + it('should download a file and measure its size', function(done) { + var entry = { + method: 'GET', + url: 'http://localhost:8388/jquery1.8.3.js', + requestHeaders: { + 'User-Agent': 'something', + Referer: 'http://www.google.fr/', + Accept: '*/*' + }, + status: 200, + isJS: true + }; + + weightChecker.redownloadEntry(entry, function(err, newEntry) { + should.not.exist(err); + + newEntry.weightCheck.bodySize.should.equal(93636); + newEntry.weightCheck.uncompressedSize.should.equal(newEntry.weightCheck.bodySize); + newEntry.weightCheck.isCompressed.should.equal(false); + newEntry.weightCheck.headersSize.should.be.above(300).and.below(350); + newEntry.weightCheck.body.should.have.string('1.8.3'); + + done(); + }); + }); + + it('should fail downloading a file in error', function(done) { + var entry = { + method: 'GET', + url: 'http://localhost:8388/no-file', + requestHeaders: { + 'User-Agent': 'something', + Referer: 'http://www.google.fr/', + Accept: '*/*' + }, + status: 200, + isHTML: true, + contentLength: 999 + }; + + weightChecker.redownloadEntry(entry, function(err, newEntry) { + should.not.exist(err); + + newEntry.weightCheck.should.have.a.property('message').that.equals('error while downloading: 404'); + + done(); + }); + }); + + it('should not download a non 200 request', function(done) { + var entry = { + method: 'GET', + url: 'http://localhost:8388/no-file', + requestHeaders: { + 'User-Agent': 'something', + Referer: 'http://www.google.fr/', + Accept: '*/*' + }, + status: 302, + isHTML: true, + contentLength: 999 + }; + + weightChecker.redownloadEntry(entry, function(err, newEntry) { + should.not.exist(err); + + newEntry.weightCheck.should.have.a.property('message').that.equals('only downloading requests with status code 200'); + + done(); + }); + }); + + it('should listRequestWeight', function() { + var totalWeightObj = weightChecker.listRequestWeight([{ + method: 'GET', + url: 'http://localhost:8388/jquery1.8.3.js', + requestHeaders: { + 'User-Agent': 'something', + Referer: 'http://www.google.fr/', + Accept: '*/*', + 'Accept-Encoding': 'gzip, deflate' + }, + status: 200, + isHTML: true, + type: 'html', + contentLength: 999, + weightCheck: { + body: 'blabla', + headersSize: 200, + bodySize: 500, + isCompressed: false, + uncompressedSize: 500 + } + }]); + + totalWeightObj.totalWeight.should.equal(700); + totalWeightObj.byType.html.totalWeight.should.equal(700); + totalWeightObj.byType.html.requests.should.have.length(1); + totalWeightObj.byType.html.requests[0].should.have.a.property('url').that.equals('http://localhost:8388/jquery1.8.3.js'); + totalWeightObj.byType.html.requests[0].should.have.a.property('weight').that.equals(700); + }); + + it('should listRequestWeight even if download failed', function() { + var totalWeightObj = weightChecker.listRequestWeight([{ + method: 'GET', + url: 'http://localhost:8388/jquery1.8.3.js', + requestHeaders: { + 'User-Agent': 'something', + Referer: 'http://www.google.fr/', + Accept: '*/*', + 'Accept-Encoding': 'gzip, deflate' + }, + status: 200, + isHTML: true, + type: 'html', + contentLength: 999, + weightCheck: { + message: 'error while downloading: 404' + } + }]); + + totalWeightObj.totalWeight.should.equal(999); + }); + +});