Adding totalWeight metric
This commit is contained in:
Gaël Métais
+1
-1
@@ -143,7 +143,7 @@ module.exports = function(grunt) {
|
||||
options: {
|
||||
reporter: 'spec',
|
||||
},
|
||||
src: ['test/core/offendersHelpersTest.js']
|
||||
src: ['test/core/phantomasWrapperTest.js']
|
||||
},
|
||||
coverage: {
|
||||
options: {
|
||||
|
||||
+4
-1
@@ -3,6 +3,7 @@ var debug = require('debug')('ylt:runner');
|
||||
|
||||
var phantomasWrapper = require('./tools/phantomas/phantomasWrapper');
|
||||
var jsExecutionTransformer = require('./tools/jsExecutionTransformer');
|
||||
var weightChecker = require('./tools/weightChecker');
|
||||
var rulesChecker = require('./rulesChecker');
|
||||
var scoreCalculator = require('./scoreCalculator');
|
||||
|
||||
@@ -25,8 +26,10 @@ var Runner = function(params) {
|
||||
// Treat the JS Execution Tree from offenders
|
||||
data = jsExecutionTransformer.transform(data);
|
||||
|
||||
// Other tools go here
|
||||
// Redownload every file
|
||||
return weightChecker.recheckAllFiles(data);
|
||||
|
||||
}).then(function(data) {
|
||||
|
||||
// Rules checker
|
||||
var policies = require('./metadata/policies');
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
/**
|
||||
* Retries download on every request to get the real file size
|
||||
*
|
||||
*/
|
||||
|
||||
exports.version = '0.1';
|
||||
|
||||
exports.module = function(phantomas) {
|
||||
'use strict';
|
||||
|
||||
phantomas.setMetric('requestsList');
|
||||
|
||||
var requests = [];
|
||||
|
||||
phantomas.on('recv', function(entry, res) {
|
||||
requests.push(entry);
|
||||
});
|
||||
|
||||
phantomas.on('report', function() {
|
||||
phantomas.setMetric('requestsList', true, true);
|
||||
phantomas.addOffender('requestsList', JSON.stringify(requests));
|
||||
});
|
||||
};
|
||||
@@ -67,7 +67,7 @@ var PhantomasWrapper = function() {
|
||||
|
||||
if (value === true) {
|
||||
optionsString += ' ' + '--' + opt;
|
||||
} else if (value === false || value === null) {
|
||||
} else if (value === false || value === null || value === undefined) {
|
||||
// Nothing
|
||||
} else {
|
||||
optionsString += ' ' + '--' + opt + '=' + value;
|
||||
@@ -122,7 +122,7 @@ var PhantomasWrapper = function() {
|
||||
}
|
||||
|
||||
|
||||
debug('Returning from Phantomas');
|
||||
debug('Returning from Phantomas with error %s', err);
|
||||
|
||||
// Adding some YellowLabTools errors here
|
||||
if (json && json.metrics && (!json.metrics.javascriptExecutionTree || !json.offenders.javascriptExecutionTree)) {
|
||||
|
||||
@@ -0,0 +1,279 @@
|
||||
/*
|
||||
* Redownloading every files after Phantomas has finished
|
||||
* Checks weight and every kind of compression
|
||||
*
|
||||
*/
|
||||
|
||||
|
||||
var debug = require('debug')('ylt:weightChecker');
|
||||
|
||||
var request = require('request');
|
||||
var http = require('http');
|
||||
var async = require('async');
|
||||
var Q = require('Q');
|
||||
var zlib = require('zlib');
|
||||
|
||||
var WeightChecker = function() {
|
||||
|
||||
var MAX_PARALLEL_DOWNLOADS = 10;
|
||||
var REQUEST_TIMEOUT = 10000; // 10 seconds
|
||||
|
||||
function recheckAllFiles(data) {
|
||||
var deferred = Q.defer();
|
||||
|
||||
var requestsList = JSON.parse(data.toolsResults.phantomas.offenders.requestsList);
|
||||
delete data.toolsResults.phantomas.metrics.requestsList;
|
||||
delete data.toolsResults.phantomas.offenders.requestsList;
|
||||
|
||||
// Transform every request into a download function with a callback when done
|
||||
var redownloadList = requestsList.map(function(entry) {
|
||||
return function(callback) {
|
||||
redownloadEntry(entry, callback);
|
||||
};
|
||||
});
|
||||
|
||||
// Lanch all redownload functions and wait for completion
|
||||
async.parallelLimit(redownloadList, MAX_PARALLEL_DOWNLOADS, function(err, results) {
|
||||
if (err) {
|
||||
debug(err);
|
||||
deferred.reject(err);
|
||||
} else {
|
||||
debug('All files checked');
|
||||
|
||||
var metrics = {};
|
||||
var offenders = {};
|
||||
|
||||
|
||||
// Total weight
|
||||
offenders.totalWeight = listRequestWeight(results);
|
||||
metrics.totalWeight = offenders.totalWeight.totalWeight;
|
||||
|
||||
|
||||
data.toolsResults.weightChecker = {
|
||||
metrics: metrics,
|
||||
offenders: offenders
|
||||
};
|
||||
|
||||
deferred.resolve(data);
|
||||
}
|
||||
});
|
||||
|
||||
return deferred.promise;
|
||||
}
|
||||
|
||||
|
||||
function listRequestWeight(requests) {
|
||||
var results = {
|
||||
totalWeight: 0,
|
||||
byType: {
|
||||
html: {
|
||||
totalWeight: 0,
|
||||
requests: []
|
||||
},
|
||||
css: {
|
||||
totalWeight: 0,
|
||||
requests: []
|
||||
},
|
||||
js: {
|
||||
totalWeight: 0,
|
||||
requests: []
|
||||
},
|
||||
json: {
|
||||
totalWeight: 0,
|
||||
requests: []
|
||||
},
|
||||
image: {
|
||||
totalWeight: 0,
|
||||
requests: []
|
||||
},
|
||||
video: {
|
||||
totalWeight: 0,
|
||||
requests: []
|
||||
},
|
||||
webfont: {
|
||||
totalWeight: 0,
|
||||
requests: []
|
||||
},
|
||||
other: {
|
||||
totalWeight: 0,
|
||||
requests: []
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
requests.forEach(function(req) {
|
||||
var weight = (typeof req.weightCheck.bodySize === 'number') ? req.weightCheck.bodySize + req.weightCheck.headersSize : req.contentLength;
|
||||
var type = req.type || 'other';
|
||||
|
||||
results.totalWeight += weight;
|
||||
results.byType[type].totalWeight += weight;
|
||||
|
||||
results.byType[type].requests.push({
|
||||
url: req.url,
|
||||
weight: weight
|
||||
});
|
||||
});
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
|
||||
function redownloadEntry(entry, callback) {
|
||||
|
||||
function onError(message) {
|
||||
debug('Could not download %s Error: %s', entry.url, message);
|
||||
entry.weightCheck = {
|
||||
message: message
|
||||
};
|
||||
setImmediate(function() {
|
||||
callback(null, entry);
|
||||
});
|
||||
}
|
||||
|
||||
if (entry.method !== 'GET') {
|
||||
onError('only downloading GET');
|
||||
return;
|
||||
}
|
||||
|
||||
if (entry.status !== 200) {
|
||||
onError('only downloading requests with status code 200');
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
debug('Downloading %s', entry.url);
|
||||
|
||||
// Always add a gzip header before sending, in case the server listens to it
|
||||
var reqHeaders = entry.requestHeaders;
|
||||
reqHeaders['Accept-Encoding'] = 'gzip, deflate';
|
||||
|
||||
var requestOptions = {
|
||||
method: entry.method,
|
||||
url: entry.url,
|
||||
headers: reqHeaders,
|
||||
timeout: REQUEST_TIMEOUT
|
||||
};
|
||||
|
||||
download(requestOptions, function(error, result) {
|
||||
if (error) {
|
||||
if (error.code === 'ETIMEDOUT') {
|
||||
onError('timeout after ' + REQUEST_TIMEOUT + 'ms');
|
||||
} else {
|
||||
onError('error while downloading: ' + error.code);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
debug('%s downloaded correctly', entry.url);
|
||||
|
||||
entry.weightCheck = result;
|
||||
callback(null, entry);
|
||||
});
|
||||
}
|
||||
|
||||
// Inspired by https://github.com/cvan/fastHAR-api/blob/10cec585/app.js
|
||||
function download(requestOptions, callback) {
|
||||
|
||||
var statusCode;
|
||||
|
||||
request(requestOptions)
|
||||
|
||||
.on('response', function(res) {
|
||||
|
||||
// Raw headers were added in NodeJS v0.12
|
||||
// (https://github.com/joyent/node/issues/4844), but let's
|
||||
// reconstruct them for backwards compatibility.
|
||||
var rawHeaders = ('HTTP/' + res.httpVersion + ' ' + res.statusCode +
|
||||
' ' + http.STATUS_CODES[res.statusCode] + '\r\n');
|
||||
Object.keys(res.headers).forEach(function(headerKey) {
|
||||
rawHeaders += headerKey + ': ' + res.headers[headerKey] + '\r\n';
|
||||
});
|
||||
rawHeaders += '\r\n';
|
||||
|
||||
var uncompressedSize = 0; // size after uncompression
|
||||
var bodySize = 0; // bytes size over the wire
|
||||
var body = ''; // plain text body (after uncompressing gzip/deflate)
|
||||
var isCompressed = false;
|
||||
|
||||
function tally() {
|
||||
|
||||
if (statusCode !== 200) {
|
||||
callback({code: statusCode});
|
||||
return;
|
||||
}
|
||||
|
||||
var result = {
|
||||
body: body,
|
||||
headersSize: Buffer.byteLength(rawHeaders, 'utf8'),
|
||||
bodySize: bodySize,
|
||||
isCompressed: isCompressed,
|
||||
uncompressedSize: uncompressedSize
|
||||
};
|
||||
|
||||
callback(null, result);
|
||||
}
|
||||
|
||||
switch (res.headers['content-encoding']) {
|
||||
case 'gzip':
|
||||
var gzip = zlib.createGunzip();
|
||||
|
||||
gzip.on('data', function (data) {
|
||||
body += data;
|
||||
uncompressedSize += data.length;
|
||||
}).on('end', function () {
|
||||
isCompressed = true;
|
||||
tally();
|
||||
});
|
||||
|
||||
res.on('data', function (data) {
|
||||
bodySize += data.length;
|
||||
}).pipe(gzip);
|
||||
|
||||
break;
|
||||
case 'deflate':
|
||||
var deflate = zlib.createInflate();
|
||||
|
||||
deflate.on('data', function (data) {
|
||||
body += data;
|
||||
uncompressedSize += data.length;
|
||||
}).on('end', function () {
|
||||
isCompressed = true;
|
||||
tally();
|
||||
});
|
||||
|
||||
res.on('data', function (data) {
|
||||
bodySize += data.length;
|
||||
}).pipe(deflate);
|
||||
|
||||
break;
|
||||
default:
|
||||
res.on('data', function (data) {
|
||||
body += data;
|
||||
uncompressedSize += data.length;
|
||||
bodySize += data.length;
|
||||
}).on('end', function () {
|
||||
tally();
|
||||
});
|
||||
|
||||
break;
|
||||
}
|
||||
})
|
||||
|
||||
.on('response', function(response) {
|
||||
statusCode = response.statusCode;
|
||||
})
|
||||
|
||||
.on('error', function(err) {
|
||||
callback(err);
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
recheckAllFiles: recheckAllFiles,
|
||||
listRequestWeight: listRequestWeight,
|
||||
redownloadEntry: redownloadEntry,
|
||||
download: download
|
||||
};
|
||||
};
|
||||
|
||||
module.exports = new WeightChecker();
|
||||
@@ -42,10 +42,13 @@ describe('phantomasWrapper', function() {
|
||||
phantomasWrapper.execute({
|
||||
params: {
|
||||
url: url,
|
||||
options: {}
|
||||
options: {
|
||||
device: 'desktop'
|
||||
}
|
||||
}
|
||||
}).then(function(data) {
|
||||
|
||||
console.log(data);
|
||||
done('Error: unwanted success');
|
||||
|
||||
}).fail(function(err) {
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
var should = require('chai').should();
|
||||
var weightChecker = require('../../lib/tools/weightChecker');
|
||||
|
||||
describe('weightChecker', function() {
|
||||
|
||||
it('should download a list of files', function(done) {
|
||||
var requestsList = [
|
||||
{
|
||||
method: 'GET',
|
||||
url: 'http://localhost:8388/simple-page.html',
|
||||
requestHeaders: {
|
||||
'User-Agent': 'something',
|
||||
Referer: 'http://www.google.fr/',
|
||||
Accept: '*/*'
|
||||
},
|
||||
status: 200,
|
||||
isHTML: true
|
||||
},
|
||||
{
|
||||
method: 'GET',
|
||||
url: 'http://localhost:8388/jquery1.8.3.js',
|
||||
requestHeaders: {
|
||||
'User-Agent': 'something',
|
||||
Referer: 'http://www.google.fr/',
|
||||
Accept: '*/*'
|
||||
},
|
||||
status: 200,
|
||||
isJS: true
|
||||
}
|
||||
];
|
||||
|
||||
var data = {
|
||||
toolsResults: {
|
||||
phantomas: {
|
||||
metrics: {
|
||||
requestsList: true
|
||||
},
|
||||
offenders: {
|
||||
requestsList: JSON.stringify(requestsList)
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
weightChecker.recheckAllFiles(data)
|
||||
|
||||
.then(function(data) {
|
||||
data.toolsResults.should.have.a.property('weightChecker');
|
||||
data.toolsResults.weightChecker.should.have.a.property('metrics');
|
||||
data.toolsResults.weightChecker.should.have.a.property('offenders');
|
||||
done();
|
||||
})
|
||||
|
||||
.fail(function(err) {
|
||||
done(err);
|
||||
});
|
||||
});
|
||||
|
||||
it('should download a file and measure its size', function(done) {
|
||||
var entry = {
|
||||
method: 'GET',
|
||||
url: 'http://localhost:8388/jquery1.8.3.js',
|
||||
requestHeaders: {
|
||||
'User-Agent': 'something',
|
||||
Referer: 'http://www.google.fr/',
|
||||
Accept: '*/*'
|
||||
},
|
||||
status: 200,
|
||||
isJS: true
|
||||
};
|
||||
|
||||
weightChecker.redownloadEntry(entry, function(err, newEntry) {
|
||||
should.not.exist(err);
|
||||
|
||||
newEntry.weightCheck.bodySize.should.equal(93636);
|
||||
newEntry.weightCheck.uncompressedSize.should.equal(newEntry.weightCheck.bodySize);
|
||||
newEntry.weightCheck.isCompressed.should.equal(false);
|
||||
newEntry.weightCheck.headersSize.should.be.above(300).and.below(350);
|
||||
newEntry.weightCheck.body.should.have.string('1.8.3');
|
||||
|
||||
done();
|
||||
});
|
||||
});
|
||||
|
||||
it('should fail downloading a file in error', function(done) {
|
||||
var entry = {
|
||||
method: 'GET',
|
||||
url: 'http://localhost:8388/no-file',
|
||||
requestHeaders: {
|
||||
'User-Agent': 'something',
|
||||
Referer: 'http://www.google.fr/',
|
||||
Accept: '*/*'
|
||||
},
|
||||
status: 200,
|
||||
isHTML: true,
|
||||
contentLength: 999
|
||||
};
|
||||
|
||||
weightChecker.redownloadEntry(entry, function(err, newEntry) {
|
||||
should.not.exist(err);
|
||||
|
||||
newEntry.weightCheck.should.have.a.property('message').that.equals('error while downloading: 404');
|
||||
|
||||
done();
|
||||
});
|
||||
});
|
||||
|
||||
it('should not download a non 200 request', function(done) {
|
||||
var entry = {
|
||||
method: 'GET',
|
||||
url: 'http://localhost:8388/no-file',
|
||||
requestHeaders: {
|
||||
'User-Agent': 'something',
|
||||
Referer: 'http://www.google.fr/',
|
||||
Accept: '*/*'
|
||||
},
|
||||
status: 302,
|
||||
isHTML: true,
|
||||
contentLength: 999
|
||||
};
|
||||
|
||||
weightChecker.redownloadEntry(entry, function(err, newEntry) {
|
||||
should.not.exist(err);
|
||||
|
||||
newEntry.weightCheck.should.have.a.property('message').that.equals('only downloading requests with status code 200');
|
||||
|
||||
done();
|
||||
});
|
||||
});
|
||||
|
||||
it('should listRequestWeight', function() {
|
||||
var totalWeightObj = weightChecker.listRequestWeight([{
|
||||
method: 'GET',
|
||||
url: 'http://localhost:8388/jquery1.8.3.js',
|
||||
requestHeaders: {
|
||||
'User-Agent': 'something',
|
||||
Referer: 'http://www.google.fr/',
|
||||
Accept: '*/*',
|
||||
'Accept-Encoding': 'gzip, deflate'
|
||||
},
|
||||
status: 200,
|
||||
isHTML: true,
|
||||
type: 'html',
|
||||
contentLength: 999,
|
||||
weightCheck: {
|
||||
body: 'blabla',
|
||||
headersSize: 200,
|
||||
bodySize: 500,
|
||||
isCompressed: false,
|
||||
uncompressedSize: 500
|
||||
}
|
||||
}]);
|
||||
|
||||
totalWeightObj.totalWeight.should.equal(700);
|
||||
totalWeightObj.byType.html.totalWeight.should.equal(700);
|
||||
totalWeightObj.byType.html.requests.should.have.length(1);
|
||||
totalWeightObj.byType.html.requests[0].should.have.a.property('url').that.equals('http://localhost:8388/jquery1.8.3.js');
|
||||
totalWeightObj.byType.html.requests[0].should.have.a.property('weight').that.equals(700);
|
||||
});
|
||||
|
||||
it('should listRequestWeight even if download failed', function() {
|
||||
var totalWeightObj = weightChecker.listRequestWeight([{
|
||||
method: 'GET',
|
||||
url: 'http://localhost:8388/jquery1.8.3.js',
|
||||
requestHeaders: {
|
||||
'User-Agent': 'something',
|
||||
Referer: 'http://www.google.fr/',
|
||||
Accept: '*/*',
|
||||
'Accept-Encoding': 'gzip, deflate'
|
||||
},
|
||||
status: 200,
|
||||
isHTML: true,
|
||||
type: 'html',
|
||||
contentLength: 999,
|
||||
weightCheck: {
|
||||
message: 'error while downloading: 404'
|
||||
}
|
||||
}]);
|
||||
|
||||
totalWeightObj.totalWeight.should.equal(999);
|
||||
});
|
||||
|
||||
});
|
||||
Reference in New Issue
Block a user