Splitted the node server in modules

This commit is contained in:
Gaël Métais
2014-08-07 19:26:58 +02:00
parent 9b6b0f39f0
commit 7fcdf2aaa2
17 changed files with 407 additions and 250 deletions
+67
View File
@@ -0,0 +1,67 @@
/**
* Yellow Lab Tools main file
*/
var q = require ('q');
var phantomas = require('phantomas');
var PhantomasWrapper = function() {
'use strict';
/**
* This is the phantomas launcher. It merges user chosen options into the default options
* Available options :
*
* - timeout : in seconds (default 60)
* - jsDeepAnalysis : should we inspect subrequests in the javascript execution tree (reported durations of main tasks will be slower than usual)
*
*/
this.execute = function(task, callback) {
var options = {
// Cusomizable options
timeout: task.options.timeout || 60,
'js-deep-analysis': task.options.jsDeepAnalysis || false,
// Mandatory
reporter: 'json:pretty',
'skip-modules': [
'ajaxRequests',
'alerts',
'cacheHits',
'caching',
'console',
'cookies',
'documentHeight',
'domains',
'domComplexity',
'domMutations',
'domQueries',
'filmStrip',
'jQuery',
'jserrors',
'har',
'headers',
'localStorage',
'mainRequest',
'pageSource',
'redirects',
'requestsStats',
'screenshot',
'staticAssets',
'timeToFirst',
'waitForSelector'
].join(','),
'include-dirs': [
'phantomas_custom/core',
'phantomas_custom/modules'
].join(',')
};
// It's time to launch the test!!!
phantomas(task.url, options, callback);
};
};
module.exports = new PhantomasWrapper();
+63
View File
@@ -0,0 +1,63 @@
/**
* Creation of a queue and it's worker function
*/
var util = require('util');
var EventEmitter = require('events').EventEmitter;
var async = require('async');
var phantomasWrapper = require('./phantomasWrapper');
var testQueue = function() {
'use strict';
var currentTask = null;
var self = this;
var queue = async.queue(function(task, callback) {
currentTask = task;
console.log('Starting test ' + task.testId);
phantomasWrapper.execute(task, function(err, json, results) {
console.log('Test ' + task.testId + ' complete');
currentTask = null;
callback(err, json, results);
self.emit('queueMoving');
});
});
// Use this method to add a test to the queue
this.push = queue.push;
// Gives the position of a task in the queue
// Returns 0 if it is the current running task
// Returns -1 if not found
this.indexOf = function(testId) {
if (currentTask && currentTask.testId === testId) {
return 0;
}
var position = -1;
if (queue.length() > 0) {
queue.tasks.forEach(function(task, index) {
if (task.data.testId === testId) {
position = index + 1;
}
});
}
return position;
};
// Forward testComplete
this.testComplete = function(testId) {
self.emit('testComplete', testId);
};
};
// extend the EventEmitter class using our Radio class
util.inherits(testQueue, EventEmitter);
module.exports = new testQueue();
+23
View File
@@ -0,0 +1,23 @@
/**
* Yellow Lab Tools home page controller
*/
var async = require('async');
var fs = require ('fs');
var indexController = function(req, res) {
'use strict';
async.parallel({
htmlTemplate: function(callback) {
fs.readFile('./app/node_views/index.html', {encoding: 'utf8'}, callback);
}
}, function(err, results) {
res.setHeader('Content-Type', 'text/html');
res.send(results.htmlTemplate);
});
};
module.exports = indexController;
@@ -0,0 +1,82 @@
/**
* Controller for the test launching page (the waiting page, after the user submited a test on the index page)
*/
var async = require('async');
var fs = require ('fs');
var launchTestController = function(req, res, testQueue) {
'use strict';
// Generate test id
var testId = (Date.now()*1000 + Math.round(Math.random()*1000)).toString(36);
var resultsPath = 'results/' + testId;
var phantomasResultsPath = resultsPath + '/results.json';
var url = req.body.url;
if (url.indexOf('http://') !== 0 && url.indexOf('https://') !== 0) {
url = 'http://' + url;
}
var options = {};
if (req.body.timeout) {
options.timeout = req.body.timeout;
}
async.waterfall([
function htmlTemplate(callback) {
fs.readFile('./app/node_views/launchTest.html', {encoding: 'utf8'}, callback);
},
function sendResponse(html, callback) {
html = html.replace('%%TEST_URL%%', url);
html = html.replace('%%TEST_ID%%', testId);
res.setHeader('Content-Type', 'text/html');
res.send(html);
callback();
},
function createFolder(callback) {
// Create results folder
fs.mkdir(resultsPath, callback);
},
function executePhantomas(callback) {
console.log('Adding test ' + testId + ' on ' + url + ' to the queue');
var task = {
testId: testId,
url: url,
options: options
};
testQueue.push(task, callback);
},
function writeResults(json, resultsObject, callback) {
console.log('Saving Phantomas results file to ' + phantomasResultsPath);
fs.writeFile(phantomasResultsPath, JSON.stringify(json, null, 4), callback);
}
], function(err) {
if (err) {
console.log('An error occured while launching the phantomas test : ', err);
fs.writeFile(phantomasResultsPath, JSON.stringify({url: url, error: err}, null, 4), function(err) {
if (err) {
console.log('Could not even write an error message on file ' + phantomasResultsPath);
console.log(err);
}
});
} else {
testQueue.testComplete(testId);
}
});
};
module.exports = launchTestController;
+41
View File
@@ -0,0 +1,41 @@
/**
* The page that dispays the results
*/
var async = require('async');
var fs = require ('fs');
var resultsController = function(req, res) {
'use strict';
var testId = req.params.testId;
var resultsPath = 'results/' + testId;
var phantomasResultsPath = resultsPath + '/results.json';
console.log('Opening test ' + testId + ' results as HTML');
async.parallel({
htmlTemplate: function(callback) {
fs.readFile('./app/node_views/results.html', {encoding: 'utf8'}, callback);
},
phantomasResults: function(callback) {
fs.readFile(phantomasResultsPath, {encoding: 'utf8'}, callback);
}
}, function(err, results) {
if (err) {
console.log(err);
return res.status(404).send('Sorry, test not found...');
}
var html = results.htmlTemplate;
html = html.replace('%%RESULTS%%', results.phantomasResults);
res.setHeader('Content-Type', 'text/html');
res.send(html);
});
};
module.exports = resultsController;
@@ -0,0 +1,53 @@
/**
* Socket.io handler
*/
var fs = require('fs');
var waitingQueueSocket = function(socket, testQueue) {
socket.on('waiting', function(testId) {
console.log('User waiting for test id ' + testId);
sendTestStatus(testId);
testQueue.on('testComplete', function(id) {
if (testId === id) {
socket.emit('complete');
console.log('Sending complete event to test id ' + testId);
}
});
testQueue.on('queueMoving', function() {
var positionInQueue = testQueue.indexOf(testId);
if (positionInQueue >= 0) {
socket.emit('position', positionInQueue);
console.log('Sending position to test id ' + testId);
}
});
});
// Finds the status of a test and send it to the client
function sendTestStatus(testId) {
// Check task position in queue
var positionInQueue = testQueue.indexOf(testId);
if (positionInQueue >= 0) {
socket.emit('position', positionInQueue);
} else {
// Find in results files
var exists = fs.exists('results/' + testId + '/results.json', function(exists) {
if (exists) {
// TODO : find a way to make sure the file is completely written
setTimeout(function() {
socket.emit('complete');
}, 4000);
} else {
socket.emit('404');
}
});
}
}
};
module.exports = waitingQueueSocket;
@@ -21,15 +21,13 @@
function askStatus() {
socket.emit('waiting', testId);
}
socket.on('running', function() {
statusElement.innerHTML = 'Running';
setTimeout(askStatus, 200);
});
socket.on('position', function(position) {
statusElement.innerHTML = 'Waiting behind ' + (position + 1) + ' other tests';
setTimeout(askStatus, 2000);
if (position === 0) {
statusElement.innerHTML = 'Running';
} else {
statusElement.innerHTML = 'Waiting behind ' + (position) + ' other tests';
}
});
socket.on('complete', function() {
@@ -16,7 +16,7 @@
<div ng-if="undefined">Untangling and counting the spaghettis...</div>
<div class="ng-cloak">
<div>Tested url: <a class="testedUrl" href="phantomasResults.url" target="_blank">{{phantomasResults.url}}</a></div>
<div>Tested url: <a class="testedUrl" href="{{phantomasResults.url}}" target="_blank">{{phantomasResults.url}}</a></div>
<div ng-if="phantomasResults.error || !javascript">
<h2>Error: {{phantomasResults.error}}</h2>
@@ -74,7 +74,7 @@
ng-class="{
'warningDetails': node.data.type == 'jQuery - bind' && node.data.callDetails.context.length > 5
}"
ng-if="node.data.type != 'script loaded'">i</button>
ng-if="node.data.type != 'script loaded' && node.data.type != 'jQuery loaded'">i</button>
<div class="detailsOverlay" ng-show="node.data.showDetails">
<div class="closeBtn" ng-click="onNodeDetailsClick(node)"></div>
@@ -125,14 +125,17 @@
<span ng-if="node.data.callDetails.arguments.length > 2"> : {{node.data.callDetails.arguments[2]}}</span>
<span ng-if="node.data.callDetails.arguments.length > 3"> : {{node.data.callDetails.arguments[3]}}</span>
</div>
<div class="duration" ng-if="node.data.time != undefined">{{node.data.time}} ms <div ng-if="node.data.time > slowRequestsLimit" class="warningIcon"></div></div>
<div class="duration" ng-if="node.data.time != undefined">{{node.data.time}} ms</div>
<div class="duration" ng-if="node.data.time == undefined"></div>
</div>
</div>
</div>
</div>
</div>
<div class="duration" ng-if="node.data.time != undefined">{{node.data.time}} ms <div ng-if="node.data.time > slowRequestsLimit" class="warningIcon"></div></div>
<div class="duration" ng-if="node.data.time != undefined">
{{node.data.time}} ms
<div ng-if="node.data.time > slowRequestsLimit" class="warningIcon" title="Slower than {{slowRequestsLimit}} ms"></div>
</div>
<div class="duration" ng-if="node.data.time == undefined"></div>
</div>
</div>
+4 -6
View File
@@ -6,11 +6,11 @@
* @see http://code.jquery.com/jquery-2.0.3.js
*/
/* global document: true, window: true */
'use strict';
exports.version = '0.1';
exports.module = function(phantomas) {
'use strict';
phantomas.once('init', function() {
phantomas.evaluate(function(deepAnalysis) {
@@ -43,8 +43,7 @@ exports.module = function(phantomas) {
// Before
if (enabled) {
var args = Array.prototype.slice.call(arguments);
callbackBefore.apply(this, args);
callbackBefore.apply(this, arguments);
}
// Execute
@@ -56,8 +55,7 @@ exports.module = function(phantomas) {
// After
if (enabled && callbackAfter) {
var args = Array.prototype.slice.call(arguments);
callbackAfter.apply(this, args);
callbackAfter.apply(this, arguments);
}
}
@@ -158,7 +156,7 @@ exports.module = function(phantomas) {
var child = new ContextTreeNode(this, data);
this.children.push(child);
return child;
}
};
}
phantomas.log('Adding some contextTree functions to phantomas');
@@ -2,11 +2,11 @@
* Analyzes DOM complexity
*/
/* global document: true, Node: true, window: true */
'use strict';
exports.version = '0.3.a';
exports.module = function(phantomas) {
'use strict';
// total length of HTML comments (including <!-- --> brackets)
phantomas.setMetric('commentsSize'); // @desc the size of HTML comments on the page @offenders
+23 -9
View File
@@ -2,18 +2,19 @@
* Analyzes DOM queries done via native DOM methods
*/
/* global Element: true, Document: true, Node: true, window: true */
'use strict';
exports.version = '0.7.a';
exports.module = function(phantomas) {
phantomas.setMetric('DOMqueries'); // @desc number of all DOM queries @offenders
phantomas.setMetric('DOMqueriesById'); // @desc number of document.getElementById calls
phantomas.setMetric('DOMqueriesByClassName'); // @desc number of document.getElementsByClassName calls
phantomas.setMetric('DOMqueriesByTagName'); // @desc number of document.getElementsByTagName calls
phantomas.setMetric('DOMqueriesByQuerySelectorAll'); // @desc number of document.querySelector(All) calls
phantomas.setMetric('DOMinserts'); // @desc number of DOM nodes inserts
phantomas.setMetric('DOMqueriesDuplicated'); // @desc number of duplicated DOM queries
'use strict';
phantomas.setMetric('DOMqueries'); // @desc number of all DOM queries @offenders
phantomas.setMetric('DOMqueriesById'); // @desc number of document.getElementById calls
phantomas.setMetric('DOMqueriesByClassName'); // @desc number of document.getElementsByClassName calls
phantomas.setMetric('DOMqueriesByTagName'); // @desc number of document.getElementsByTagName calls
phantomas.setMetric('DOMqueriesByQuerySelectorAll'); // @desc number of document.querySelector(All) calls
phantomas.setMetric('DOMinserts'); // @desc number of DOM nodes inserts
phantomas.setMetric('DOMqueriesDuplicated'); // @desc number of duplicated DOM queries
// fake native DOM functions
phantomas.once('init', function() {
@@ -43,6 +44,8 @@ exports.module = function(phantomas) {
// selectors by class name
function selectorClassNameSpyBefore(className) {
/*jshint validthis: true */
phantomas.incrMetric('DOMqueriesByClassName');
phantomas.addOffender('DOMqueriesByClassName', '.' + className);
querySpy('class', '.' + className, 'getElementsByClassName');
@@ -65,6 +68,8 @@ exports.module = function(phantomas) {
// selectors by tag name
function selectorTagNameSpyBefore(tagName) {
/*jshint validthis: true */
phantomas.incrMetric('DOMqueriesByTagName');
phantomas.addOffender('DOMqueriesByTagName', tagName);
querySpy('tag name', tagName, 'getElementsByTagName');
@@ -93,6 +98,8 @@ exports.module = function(phantomas) {
}
function selectorQuerySpyBefore(selector) {
/*jshint validthis: true */
selectorQuerySpy(selector);
phantomas.enterContext({
@@ -109,6 +116,8 @@ exports.module = function(phantomas) {
}
function selectorAllQuerySpyBefore(selector) {
/*jshint validthis: true */
selectorQuerySpy(selector);
phantomas.enterContext({
@@ -132,7 +141,8 @@ exports.module = function(phantomas) {
// count DOM inserts
function appendChild(child) {
/* jshint validthis: true */
/*jshint validthis: true */
// ignore appending to the node that's not yet added to DOM tree
if (!this.parentNode) {
return;
@@ -154,6 +164,8 @@ exports.module = function(phantomas) {
}
function appendChildSpyBefore(child) {
/*jshint validthis: true */
appendChild(child);
phantomas.enterContext({
@@ -170,6 +182,8 @@ exports.module = function(phantomas) {
}
function insertBeforeSpyBefore(child) {
/*jshint validthis: true */
appendChild(child);
phantomas.enterContext({
+10 -8
View File
@@ -5,17 +5,19 @@
* @see http://code.jquery.com/jquery-2.0.3.js
*/
/* global document: true, window: true */
'use strict';
/* jshint -W030 */
exports.version = '0.2.a';
exports.module = function(phantomas) {
phantomas.setMetric('jQueryVersion', ''); // @desc version of jQuery framework (if loaded) [string]
phantomas.setMetric('jQueryOnDOMReadyFunctions'); // @desc number of functions bound to onDOMReady event
phantomas.setMetric('jQuerySizzleCalls'); // @desc number of calls to Sizzle (including those that will be resolved using querySelectorAll)
phantomas.setMetric('jQuerySizzleCallsDuplicated'); // @desc number of calls on the same Sizzle request
phantomas.setMetric('jQueryBindOnMultipleElements'); //@desc number of calls to jQuery bind function on 2 or more elements
phantomas.setMetric('jQueryDifferentVersions'); //@desc number of different jQuery versions loaded on the page (not counting iframes)
'use strict';
phantomas.setMetric('jQueryVersion', ''); // @desc version of jQuery framework (if loaded) [string]
phantomas.setMetric('jQueryOnDOMReadyFunctions'); // @desc number of functions bound to onDOMReady event
phantomas.setMetric('jQuerySizzleCalls'); // @desc number of calls to Sizzle (including those that will be resolved using querySelectorAll)
phantomas.setMetric('jQuerySizzleCallsDuplicated'); // @desc number of calls on the same Sizzle request
phantomas.setMetric('jQueryBindOnMultipleElements'); //@desc number of calls to jQuery bind function on 2 or more elements
phantomas.setMetric('jQueryDifferentVersions'); //@desc number of different jQuery versions loaded on the page (not counting iframes)
var jQueryFunctions = [
// DOM manipulations
@@ -244,7 +246,7 @@ exports.module = function(phantomas) {
arg = 'false';
}
if (arg == null) {
if (arg === null) {
arg = 'null';
}
@@ -1,11 +1,12 @@
/**
* Meters the number of page errors, and provides traces as offenders for "jsErrors" metric
*/
'use strict';
exports.version = '0.3.a';
exports.module = function(phantomas) {
'use strict';
phantomas.setMetric('jsErrors'); // @desc number of JavaScript errors
function formatTrace(trace) {
@@ -1,11 +1,11 @@
/**
* Meters the number of page errors, and provides traces as offenders for "jsErrors" metric
*/
'use strict';
exports.version = '0.0';
exports.module = function(phantomas) {
'use strict';
phantomas.on('recv', function(entry, res) {
if (!entry.isJS) {
@@ -3,11 +3,11 @@
*
* Run phantomas with --js-execution-tree option to use this module
*/
'use strict';
exports.version = '0.1';
exports.module = function(phantomas) {
'use strict';
phantomas.setMetric('javascriptExecutionTree'); // @desc number of duplicated DOM queries
+24 -212
View File
@@ -1,230 +1,42 @@
var settings = require('./server_config/settings.json');
var fs = require('fs');
var async = require('async');
var express = require('express');
var app = express();
var server = require('http').createServer(app);
var io = require('socket.io').listen(server);
var bodyParser = require('body-parser');
var phantomas = require('phantomas');
// Config file
var settings = require('./server_config/settings.json');
// Libraries
var fs = require('fs');
var async = require('async');
var express = require('express');
var app = express();
var server = require('http').createServer(app);
var io = require('socket.io').listen(server);
var bodyParser = require('body-parser');
// Internals
var indexController = require('./app/node_controllers/indexController');
var launchTestController = require('./app/node_controllers/launchTestController');
var resultsController = require('./app/node_controllers/resultsController');
var waitingQueueSocket = require('./app/node_controllers/waitingQueueSocket');
var testQueue = require('./app/lib/testQueue');
app.use(bodyParser.urlencoded({ extended: false }));
// Home page
app.get('/', function(req, res) {
async.parallel({
htmlTemplate: function(callback) {
fs.readFile('./app/views/index.html', {encoding: 'utf8'}, callback);
}
}, function(err, results) {
res.setHeader('Content-Type', 'text/html');
res.send(results.htmlTemplate);
});
});
// Waiting queue page
app.post('/launchTest', function(req, res) {
// Generate test id
var testId = (Date.now()*1000 + Math.round(Math.random()*1000)).toString(36);
var resultsPath = 'results/' + testId;
var phantomasResultsPath = resultsPath + '/results.json';
var url = req.body.url;
if (url.indexOf('http://') !== 0 && url.indexOf('https://') !== 0) {
url = 'http://' + url;
}
async.waterfall([
function htmlTemplate(callback) {
fs.readFile('./app/views/launchTest.html', {encoding: 'utf8'}, callback);
},
function sendResponse(html, callback) {
html = html.replace('%%TEST_URL%%', url);
html = html.replace('%%TEST_ID%%', testId);
res.setHeader('Content-Type', 'text/html');
res.send(html);
callback();
},
function createFolder(callback) {
// Create results folder
fs.mkdir(resultsPath, callback);
},
function executePhantomas(callback) {
var options = {
timeout: 60,
'js-deep-analysis': false,
reporter: 'json:pretty',
'skip-modules': [
'ajaxRequests',
'alerts',
'cacheHits',
'caching',
'console',
'cookies',
'documentHeight',
'domains',
'domComplexity',
'domMutations',
'domQueries',
'filmStrip',
'jQuery',
'jserrors',
'har',
'headers',
'localStorage',
'mainRequest',
'pageSource',
'redirects',
'requestsStats',
'screenshot',
'staticAssets',
'timeToFirst',
'waitForSelector'
].join(','),
'include-dirs': [
'phantomas_custom/core',
'phantomas_custom/modules'
].join(',')
};
console.log('Adding test ' + testId + ' on ' + url + ' to the queue');
var task = {
testId: testId,
url: url,
options: options
};
console.log(JSON.stringify(task, null, 4));
taskQueue.push(task, callback);
},
function writeResults(json, resultsObject, callback) {
console.log('Saving Phantomas results file to ' + phantomasResultsPath);
fs.writeFile(phantomasResultsPath, JSON.stringify(json, null, 4), callback);
}
], function(err) {
if (err) {
console.log('An error occured while launching the phantomas test : ', err);
fs.writeFile(phantomasResultsPath, JSON.stringify({url: url, error: err}, null, 4), function(err) {
if (err) {
console.log('Could not even write an error message on file ' + phantomasResultsPath);
console.log(err);
}
});
}
});
});
// Results page
app.get('/results/:testId', function(req, res) {
var testId = req.params.testId;
var resultsPath = 'results/' + testId;
var phantomasResultsPath = resultsPath + '/results.json';
console.log('Opening test ' + testId + ' results as HTML');
async.parallel({
htmlTemplate: function(callback) {
fs.readFile('./app/views/results.html', {encoding: 'utf8'}, callback);
},
phantomasResults: function(callback) {
fs.readFile(phantomasResultsPath, {encoding: 'utf8'}, callback);
}
}, function(err, results) {
if (err) {
console.log(err);
return res.status(404).send('Sorry, test not found...');
}
var html = results.htmlTemplate;
html = html.replace('%%RESULTS%%', results.phantomasResults);
res.setHeader('Content-Type', 'text/html');
res.send(html);
});
});
// Routes definition
app.get('/', indexController);
app.post('/launchTest', function(req, res) { launchTestController(req, res, testQueue); });
app.get('/results/:testId', resultsController);
// Static files
app.use('/public', express.static(__dirname + '/app/public'));
app.use('/bower_components', express.static(__dirname + '/bower_components'));
app.use('/public', express.static(__dirname + '/app/public'));
app.use('/bower_components', express.static(__dirname + '/bower_components'));
// Socket.io
io.on('connection', function(socket){
socket.on('waiting', function(testId) {
console.log('User waiting for test id ' + testId);
socket.testId = testId;
// Check task position in queue
var positionInQueue = -1;
if (taskQueue.length() > 0) {
taskQueue.tasks.forEach(function(task, index) {
if (task.data.testId === testId) {
positionInQueue = index;
}
});
}
if (positionInQueue >= 0) {
socket.emit('position', positionInQueue);
} else if (currentTask && currentTask.testId === testId) {
socket.emit('running');
} else {
// Find in results files
var exists = fs.exists('results/' + testId + '/results.json', function(exists) {
if (exists) {
// TODO : use eventEmitter to make sure the file is completly written on disk
setTimeout(function() {
socket.emit('complete');
}, 1000);
} else {
socket.emit('404');
}
});
}
});
waitingQueueSocket(socket, testQueue);
});
// Creating a queue and defining the worker function
var currentTask = null;
var taskQueue = async.queue(function queueWorker(task, callback) {
currentTask = task;
console.log('Starting test ' + task.testId);
// It's time to launch the test!!!
phantomas(task.url, task.options, function(err, json, results) {
console.log('Test ' + task.testId + ' complete');
currentTask = null;
callback(err, json, results);
});
});
// Launch the server
server.listen(settings.serverPort, function() {