WEBDIENSTE/YellowLabTools
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Work in progress: switch to Phantomas v2

Browse Source
This commit is contained in:
Gaël Métais
2019-03-14 03:27:37 +01:00
parent 5e82eea9f1
commit 517206364a
14 changed files with 248 additions and 319 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/offendersHelpers.js
+9 -17
View File
@@ -172,23 +172,15 @@ var OffendersHelpers = function() {
};
this.cssOffenderPattern = function(offender) {
// Remove any line breaks
offender = offender.replace(/(\r\n|\n|\r)/gm, '');
var parts = /^(.*) (?:<([^ \(]*)>|\[inline CSS\]) ?@ ?(\d+):(\d+)$/.exec(offender);
if (!parts) {
return {
offender: offender
};
} else {
return {
css: parts[1],
file: parts[2] || null,
line: parseInt(parts[3], 10),
column: parseInt(parts[4], 10)
};
}
// Used to work with strings
// As of Phantomas v2, offender is now in JSON format.
// Let's just adapt this for now and we'll see later if we remove completely this function
return {
css: offender.value.message,
file: offender.url,
line: offender.value.position.start.line,
column: offender.value.position.start.column
};
};
this.fileWithSizePattern = function(fileWithSize) {
lib/runner.js
-6
View File
@@ -56,12 +56,6 @@ var Runner = function(params) {
generic : scoreCalculator.calculate(data, scoreProfileGeneric)
};
delete data.toolsResults.phantomas.metrics.javascriptExecutionTree;
delete data.toolsResults.phantomas.offenders.javascriptExecutionTree;
delete data.toolsResults.phantomas.metrics.scrollExecutionTree;
delete data.toolsResults.phantomas.offenders.scrollExecutionTree;
if (data.toolsResults.phantomas.offenders.blockedRequests) {
data.blockedRequests = data.toolsResults.phantomas.offenders.blockedRequests;
lib/tools/phantomas/phantomasWrapper.js
+43 -78
View File
@@ -87,91 +87,55 @@ var PhantomasWrapper = function() {
debug('node node_modules/phantomas/bin/phantomas.js --url=' + task.url + optionsString + ' --verbose');
var phantomasPid;
var isKilled = false;
// Kill phantomas if nothing happens
var killer = setTimeout(function() {
console.log('Killing phantomas because the test on ' + task.url + ' was launched ' + 5*options.timeout + ' seconds ago');
if (phantomasPid) {
ps.kill(phantomasPid, function(err) {
if (err) {
debug('Could not kill Phantomas process %s', phantomasPid);
// Suicide
process.exit(1);
// If in server mode, forever will restart the server
}
debug('Phantomas process %s was correctly killed', phantomasPid);
// Then mark the test as failed
// Error 1003 = Phantomas not answering
deferred.reject(1003);
isKilled = true;
});
} else {
// Suicide
process.exit(1);
}
}, 5*options.timeout*1000);
// It's time to launch the test!!!
var triesNumber = 2;
var currentTry = 0;
const promise = phantomas(task.url, {
'analyze-css': true
});
async.retry(triesNumber, function(cb) {
// handle the promise
promise.
then(results => {
var json = {
generator: results.getGenerator(),
url: results.getUrl(),
metrics: results.getMetrics(),
offenders: results.getAllOffenders()
};
currentTry ++;
var process = phantomas(task.url, options, function(err, json, results) {
var errorCode = err ? parseInt(err.message, 10) : null;
if (isKilled) {
debug('Process was killed, too late Phantomas, sorry...');
return;
}
debug('Returning from Phantomas with error %s', errorCode);
// Adding some YellowLabTools errors here
if (json && json.metrics && (!json.metrics.javascriptExecutionTree || !json.offenders.javascriptExecutionTree)) {
errorCode = 1001;
}
if (!errorCode && (!json || !json.metrics)) {
errorCode = 1002;
}
// Don't cancel test if it is a timeout and we've got some results
if (errorCode === 252 && json) {
debug('Timeout after ' + options.timeout + ' seconds. But it\'s not a problem, the test is valid.');
errorCode = null;
}
if (errorCode) {
debug('Attempt failed. Error code ' + errorCode);
}
cb(errorCode, json);
}).fail(function() {
// This function is useless, but the failing promise needs to be handled,
// otherwise the module meow writes in the console in case of a timeout (error code 252).
debug('Failing promise handled');
deferred.resolve(json);
}).
catch(res => {
console.error(res);
deferred.reject('Phantomas failed: ' + res.message);
});
phantomasPid = process.pid;
/*var process = phantomas(task.url, options, function(err, json, results) {
var errorCode = err ? parseInt(err.message, 10) : null;
debug('Returning from Phantomas with error %s', errorCode);
// Adding some YellowLabTools errors here
if (json && json.metrics && (!json.metrics.javascriptExecutionTree || !json.offenders.javascriptExecutionTree)) {
errorCode = 1001;
}
if (!errorCode && (!json || !json.metrics)) {
errorCode = 1002;
}
// Don't cancel test if it is a timeout and we've got some results
if (errorCode === 252 && json) {
debug('Timeout after ' + options.timeout + ' seconds. But it\'s not a problem, the test is valid.');
errorCode = null;
}
if (errorCode) {
debug('Attempt failed. Error code ' + errorCode);
}
}, function(err, json) {
clearTimeout(killer);
if (err) {
debug('All ' + triesNumber + ' attemps failed for the test');
deferred.reject(err);
@@ -182,6 +146,7 @@ var PhantomasWrapper = function() {
}
});
*/
return deferred.promise;
};
lib/tools/redownload/contentTypeChecker.js
+48
View File
@@ -4,6 +4,7 @@ var isJpg = require('is-jpg');
var isPng = require('is-png');
var isSvg = require('is-svg');
var isGif = require('is-gif');
var isWebp = require('is-webp');
var isWoff = require('is-woff');
var isWoff2 = require('is-woff2');
var isOtf = require('is-otf');
@@ -17,6 +18,39 @@ var ContentTypeChecker = function() {
debug('Entering contentTypeChecker');
// Setting isSomething values:
switch(entry.type) {
case 'html':
entry.isHTML = true;
break;
case 'xml':
entry.isXML = true;
break;
case 'css':
entry.isCSS = true;
break;
case 'js':
entry.isJS = true;
break;
case 'json':
entry.isJSON = true;
break;
case 'image':
entry.isImage = true;
break;
case 'webfont':
entry.isWebFont = true;
break;
case 'video':
entry.isVideo = true;
break;
case 'favicon':
entry.isFavicon = true;
break;
}
// Now let's check for mistakes by analysing body content. It happens more often then we think!
// Ignore very small files as they are generally tracking pixels
if (entry.weightCheck && entry.weightCheck.bodyBuffer && entry.weightCheck.bodySize > 100) {
var foundType;
@@ -24,6 +58,9 @@ var ContentTypeChecker = function() {
try {
foundType = findContentType(entry.weightCheck.bodyBuffer);
// Note: as of Phantomas v2, entry.contentType is always undefined
// It could change, so let's keep the following code in place:
if (!entry.contentType || entry.contentType === '') {
if (foundType === null) {
debug('ContentType is empty for file %s', entry.url);
@@ -71,6 +108,10 @@ var ContentTypeChecker = function() {
return contentTypes.gif;
}
if (isWebp(bodyBuffer)) {
return contentTypes.webp;
}
if (isWoff(bodyBuffer)) {
return contentTypes.woff;
}
@@ -142,6 +183,13 @@ var ContentTypeChecker = function() {
entry.isImage = true;
}
},
webp: {
mimes: ['image/webp'],
updateFn: function(entry) {
entry.type = 'image';
entry.isImage = true;
}
},
woff: {
mimes: ['application/x-font-woff', 'application/font-woff', 'font/woff'],
updateFn: function(entry) {
lib/tools/redownload/redownload.js
+66 -9
View File
@@ -34,9 +34,13 @@ var Redownload = function() {
debug('Redownload started');
var deferred = Q.defer();
var requestsList = JSON.parse(data.toolsResults.phantomas.offenders.requestsList);
delete data.toolsResults.phantomas.metrics.requestsList;
delete data.toolsResults.phantomas.offenders.requestsList;
var requestsOffenders = data.toolsResults.phantomas.offenders.requests;
var gzipOffenders = data.toolsResults.phantomas.offenders.gzipRequests;
var postOffenders = data.toolsResults.phantomas.offenders.postRequests;
var notFoundOffenders = data.toolsResults.phantomas.offenders.notFound;
var redirectOffenders = data.toolsResults.phantomas.offenders.redirects;
var requestsList = mergeOffenders(requestsOffenders, gzipOffenders, postOffenders, notFoundOffenders, redirectOffenders);
var httpAuth = null;
if (data.params && data.params.options && data.params.options.authUser && data.params.options.authPass) {
@@ -194,6 +198,57 @@ var Redownload = function() {
return deferred.promise;
}
function mergeOffenders(requests, compressedOffenders, postOffenders, notFoundOffenders, redirectOffenders) {
// Parse each request and check if it can be found in other offenders
requests.forEach(function(request) {
// Is it compressed?
if (compressedOffenders) {
compressedOffenders.some(function(entry) {
if (entry.url === request.url) {
request.compressed = true;
request.bodySize = entry.bodySize;
request.transferedSize = entry.transferedSize;
return true;
}
});
}
// Is it a POST request?
if (postOffenders) {
postOffenders.some(function(url) {
if (url === request.url) {
request.post = true;
return true;
}
});
}
// Is it a 404?
if (notFoundOffenders) {
notFoundOffenders.some(function(url) {
if (url === request.url) {
request.notFound = true;
return true;
}
});
}
// Is it a redirection?
if (redirectOffenders) {
redirectOffenders.some(function(message) {
if (message.split(' ')[0] === request.url) {
request.redirect = true;
return true;
}
});
}
});
return requests;
}
function listIncorrectContentTypes(requests) {
var results = [];
@@ -639,13 +694,15 @@ var Redownload = function() {
deferred.resolve(entry);
}
if (entry.method !== 'GET') {
if (entry.post) {
notDownloadableFile('only downloading GET');
// ... at least trying to
return deferred.promise;
}
if (entry.status !== 200) {
if (entry.notFound || entry.redirect) {
unwantedFile('only downloading requests with status code 200');
// ...at least trying to
return deferred.promise;
}
@@ -654,12 +711,12 @@ var Redownload = function() {
return deferred.promise;
}
debug('Downloading %s', entry.url);
// Always add a gzip header before sending, in case the server listens to it
var reqHeaders = entry.requestHeaders;
reqHeaders['Accept-Encoding'] = 'gzip, deflate';
// Always add gzip and webp headers before sending, in case the server listens to them
var reqHeaders = [];
reqHeaders['Accept'] = '*/*,image/webp';
reqHeaders['Accept-Encoding'] = 'gzip, deflate, br';
reqHeaders['Connection'] = 'keep-alive';
var requestOptions = {