For each domain met, the browser needs to make a DNS look-up, which is slow. Avoid having to many different domains and the page should render faster.
By the way, domain sharding is not a good practice anymore.
", "isOkThreshold": 10, "isBadThreshold": 25, - "isAbnormalThreshold": 40 + "isAbnormalThreshold": 50 } } \ No newline at end of file diff --git a/lib/server/controllers/apiController.js b/lib/server/controllers/apiController.js index fb9d7c6..ee4e0de 100644 --- a/lib/server/controllers/apiController.js +++ b/lib/server/controllers/apiController.js @@ -1,86 +1,152 @@ -var debug = require('debug')('ylt:server'); +var debug = require('debug')('ylt:server'); -var runsQueue = require('../datastores/runsQueue'); -var runsDatastore = require('../datastores/runsDatastore'); +var YellowLabTools = require('../../yellowlabtools'); +var RunsQueue = require('../datastores/runsQueue'); +var RunsDatastore = require('../datastores/runsDatastore'); +var ResultsDatastore = require('../datastores/resultsDatastore'); -function ApiController(app) { +var ApiController = function(app) { 'use strict'; + var queue = new RunsQueue(); + var runsDatastore = new RunsDatastore(); + var resultsDatastore = new ResultsDatastore(); + // Retrieve the list of all runs - /*app.get('/runs', function(req, res) { + /*app.get('/api/runs', function(req, res) { // NOT YET });*/ // Create a new run - app.post('/runs', function(req, res) { + app.post('/api/runs', function(req, res) { - // Grab the test parameters + // Grab the test parameters and generate a random run ID var run = { - // Generate a random run ID - _id: (Date.now()*1000 + Math.round(Math.random()*1000)).toString(36), + runId: (Date.now()*1000 + Math.round(Math.random()*1000)).toString(36), params: { url: req.body.url, - waitForResponse: req.body.waitForResponse || true + waitForResponse: req.body.waitForResponse !== false } }; // Add test to the testQueue - debug('Adding test %s to the queue', run._id); - var queuing = runsQueue.push(run._id); + debug('Adding test %s to the queue', run.runId); + var queuePromise = queue.push(run.runId); + - // Save the run to the datastore - var position = runsQueue.getPosition(run._id); - run.status = { - statusCode: (position === 0) ? STATUS_RUNNING : STATUS_AWAITING, - position: position - }; - runsDatastore.add(run); + runsDatastore.add(run, queuePromise.startingPosition); // Listening for position updates - queuing.progress(function(position) { - var savedRun = runsDatastore.get(run._id); - savedRun.status = { - statusCode: STATUS_AWAITING, - position: position - }; - runsDatastore.update(savedRun); + queuePromise.progress(function(position) { + runsDatastore.updatePosition(run.runId, position); }); + // Let's start the run + queuePromise.then(function() { - queuing.then(function() { - + runsDatastore.updatePosition(run.runId, 0); + + debug('Launching test %s on %s', run.runId, run.params.url); + + new YellowLabTools(run.params.url) + .then(function(data) { + + debug('Success'); + runsDatastore.markAsComplete(run.runId); + + // Save result in datastore + data.runId = run.runId; + resultsDatastore.saveResult(data); + + // Send result if the user was waiting + if (run.params.waitForResponse) { + + res.redirect(302, '/api/results/' + run.runId); + } + + }).fail(function(err) { + + console.error('Test failed for %s', run.params.url); + console.error(err); + console.error(err.stack); + + runsDatastore.markAsFailed(run.runId); + + }).finally(function() { + queue.remove(run.runId); + }); + + }).fail(function(err) { + console.error('Error or YLT\'s core instanciation'); + console.error(err); + console.error(err.stack); }); - // The user doesn't not want to wait for the response - if (!params.waitForResponse) { - - // Sending just the test id - res.setHeader('Content-Type', 'application/javascript'); - res.send(JSON.stringify({ - testId: testId - })); + // The user doesn't not want to wait for the response, sending the run ID only + if (!run.params.waitForResponse) { + console.log('Sending response without waiting.'); + res.setHeader('Content-Type', 'application/json'); + res.send(JSON.stringify({runId: run.runId})); } }); // Retrive one run by id - app.get('/run/:id', function(req, res) { + app.get('/api/runs/:id', function(req, res) { + var runId = req.params.id; + var run = runsDatastore.get(runId); + + if (run) { + res.setHeader('Content-Type', 'application/json'); + res.send(JSON.stringify(run, null, 2)); + } else { + res.status(404).send('Not found'); + } }); // Delete one run by id - /*app.delete('/run/:id', function(req, res) { - // NOT YET + /*app.delete('/api/runs/:id', function(req, res) { + deleteRun() });*/ + // Delete all + /*app.delete('/api/runs', function(req, res) { + purgeRuns() + }); - var STATUS_AWAITING = 'awaiting'; - var STATUS_RUNNING = 'running'; - var STATUS_DONE = 'done'; - var STATUS_FAILED = 'failed'; + // List all + app.get('/api/runs', function(req, res) { + listRuns() + }); -} + // Exists + app.head('/api/runs/:id', function(req, res) { + existsX(); + // Retourne 200 si existe ou 404 si n'existe pas + }); + */ + + // Retrive one result by id + app.get('/api/results/:id', function(req, res) { + var runId = req.params.id; + + resultsDatastore.getResult(runId) + .then(function(data) { + // This is the pivot format, we might need to clean it first? + + // Hide phantomas results + data.toolsResults.phantomas = {}; + + res.setHeader('Content-Type', 'application/json'); + res.send(JSON.stringify(data, null, 2)); + }).fail(function() { + res.status(404).send('Not found'); + }); + }); + +}; module.exports = ApiController; \ No newline at end of file diff --git a/lib/server/controllers/uiController.js b/lib/server/controllers/uiController.js index e69de29..a6d906d 100644 --- a/lib/server/controllers/uiController.js +++ b/lib/server/controllers/uiController.js @@ -0,0 +1,13 @@ +var UiController = function(app) { + 'use strict'; + + + // Create a new run + app.get('/', function(req, res) { + + res.setHeader('Content-Type', 'text/html'); + res.send('Test'); + }); +}; + +module.exports = UiController; \ No newline at end of file diff --git a/lib/server/datastores/resultsDatastore.js b/lib/server/datastores/resultsDatastore.js index 954f9b7..3d2ce8a 100644 --- a/lib/server/datastores/resultsDatastore.js +++ b/lib/server/datastores/resultsDatastore.js @@ -1,9 +1,79 @@ +var fs = require('fs'); +var rimraf = require('rimraf'); +var path = require('path'); +var Q = require('q'); function ResultsDatastore() { 'use strict'; - + var resultFileName = 'results.json'; + var resultsFolderName = 'results'; + var resultsDir = path.join(__dirname, '..', '..', '..', resultsFolderName); + + + this.saveResult = function(testResults) { + + var promise = createResultFolder(testResults.runId); + + promise.then(function() { + + var resultFilePath = path.join(resultsDir, testResults.runId, resultFileName); + + return Q.nfcall(fs.writeFile, resultFilePath, JSON.stringify(testResults, null, 2)); + }); + + return promise; + }; + + + this.getResult = function(runId) { + + var resultFilePath = path.join(resultsDir, runId, resultFileName); + + return Q.nfcall(fs.readFile, resultFilePath, {encoding: 'utf8'}).then(function(data) { + return JSON.parse(data); + }); + }; + + + this.deleteResult = function(runId) { + var folder = path.join(resultsDir, runId); + + return Q.nfcall(rimraf, folder); + }; + + + // The folder /results/folderName/ + function createResultFolder(folderName) { + var folder = path.join(resultsDir, folderName); + + return createGlobalFolder().then(function() { + return Q.nfcall(fs.mkdir, folder); + }); + } + + // The folder /results/ + function createGlobalFolder() { + var deferred = Q.defer(); + + // Create the results folder if it doesn't exist + fs.exists(resultsDir, function(exists) { + if (exists) { + deferred.resolve(); + } else { + fs.mkdir(resultsDir, function(err) { + if (err) { + deferred.reject(err); + } else { + deferred.resolve(); + } + }); + } + }); + + return deferred.promise; + } } module.exports = ResultsDatastore; \ No newline at end of file diff --git a/lib/server/datastores/runsDatastore.js b/lib/server/datastores/runsDatastore.js index c84ac4e..cf0af10 100644 --- a/lib/server/datastores/runsDatastore.js +++ b/lib/server/datastores/runsDatastore.js @@ -7,23 +7,68 @@ function RunsDatastore() { // For the moment, maybe one day var runs = {}; + var STATUS_AWAITING = 'awaiting'; + var STATUS_RUNNING = 'running'; + var STATUS_COMPLETE = 'complete'; + var STATUS_FAILED = 'failed'; - this.add = function(run) { - runs[run._id] = run; + + this.add = function(run, position) { + runs[run.runId] = run; + this.updatePosition(run.runId, position); }; + this.get = function(runId) { return runs[runId]; }; + - this.update = function(run) { - runs[run._id] = run; + this.updatePosition = function(runId, position) { + var run = runs[runId]; + + if (position > 0) { + run.status = { + statusCode: STATUS_AWAITING, + position: position + }; + } else { + run.status = { + statusCode: STATUS_RUNNING + }; + } + + runs[runId] = run; }; + + this.markAsComplete = function(runId) { + var run = runs[runId]; + + run.status = { + statusCode: STATUS_COMPLETE + }; + + runs[runId] = run; + }; + + + this.markAsFailed = function(runId) { + var run = runs[runId]; + + run.status = { + statusCode: STATUS_FAILED + }; + + runs[runId] = run; + }; + + this.delete = function(runId) { delete runs[runId]; }; + this.list = function() { var runsArray = []; Object.keys(runs).forEach(function(key) { diff --git a/lib/server/datastores/runsQueue.js b/lib/server/datastores/runsQueue.js index ad1a6b8..26a6717 100644 --- a/lib/server/datastores/runsQueue.js +++ b/lib/server/datastores/runsQueue.js @@ -9,8 +9,9 @@ function RunsQueue() { this.push = function(runId) { var deferred = Q.defer(); + var startingPosition = queue.length; - if (queue.length === 0) { + if (startingPosition === 0) { // The queue is empty, let's run immediatly queue.push({ @@ -32,7 +33,9 @@ function RunsQueue() { }); } - return deferred.promise; + var promise = deferred.promise; + promise.startingPosition = startingPosition; + return promise; }; @@ -68,6 +71,10 @@ function RunsQueue() { }); }; + + this.length = function() { + return queue.length; + }; } module.exports = RunsQueue; \ No newline at end of file diff --git a/lib/tools/phantomasWrapper.js b/lib/tools/phantomasWrapper.js index e1df01d..adb3dd7 100644 --- a/lib/tools/phantomasWrapper.js +++ b/lib/tools/phantomasWrapper.js @@ -64,7 +64,7 @@ var PhantomasWrapper = function() { }, 600000); // It's time to launch the test!!! - var triesNumber = 3; + var triesNumber = 2; async.retry(triesNumber, function(cb) { phantomas(task.url, options, function(err, json, results) { diff --git a/lib/yellowlabtools.js b/lib/yellowlabtools.js index a0f4e38..a56e49d 100644 --- a/lib/yellowlabtools.js +++ b/lib/yellowlabtools.js @@ -14,24 +14,19 @@ var YellowLabTools = function(url, options) { } else { - // Generate a test id - var testId = (Date.now()*1000 + Math.round(Math.random()*1000)).toString(36); - if (url.indexOf('http://') !== 0 && url.indexOf('https://') !== 0) { url = 'http://' + url; } var params = { - testId: testId, url: url, options: options || {} }; - var runner = new Runner(params); - - runner.then(function(data) { - deferred.resolve(data); - }); + var runner = new Runner(params) + .then(function(data) { + deferred.resolve(data); + }); } return deferred.promise; diff --git a/package.json b/package.json index 29bca24..9ceae90 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "debug": "^2.1.0", "express": "~4.10.4", "phantomas": "1.7.0", + "rimraf": "^2.2.8", "socket.io": "~1.2.0" }, "devDependencies": { diff --git a/test/api/resultsDatastoreTest.js b/test/api/resultsDatastoreTest.js new file mode 100644 index 0000000..4e8bdaf --- /dev/null +++ b/test/api/resultsDatastoreTest.js @@ -0,0 +1,74 @@ +var should = require('chai').should(); +var resultsDatastore = require('../../lib/server/datastores/resultsDatastore'); + +describe('resultsDatastore', function() { + + var datastore = new resultsDatastore(); + + var testId1 = '123456789'; + var testData1 = { + runId: testId1, + other: { + foo: 'foo', + bar: 1 + } + }; + + + it('should store a result', function(done) { + datastore.should.have.a.property('saveResult').that.is.a('function'); + + datastore.saveResult(testData1).then(function() { + done(); + }).fail(function(err) { + done(err); + }); + }); + + it('should store another result', function(done) { + var testData2 = { + runId: '987654321', + other: { + foo: 'foo', + bar: 2 + } + }; + + datastore.saveResult(testData2).then(function() { + done(); + }).fail(function(err) { + done(err); + }); + }); + + it('should retrieve a result', function(done) { + datastore.getResult(testId1) + .then(function(results) { + + // Compare results with testData + results.should.deep.equal(testData1); + + done(); + }).fail(function(err) { + done(err); + }); + }); + + it('should delete a result', function(done) { + datastore.deleteResult(testId1) + .then(function() { + done(); + }).fail(function(err) { + done(err); + }); + }); + + it('should not find the result anymore', function(done) { + datastore.getResult(testId1) + .then(function(results) { + done('Error, the result is still in the datastore'); + }).fail(function(err) { + done(); + }); + }); +}); diff --git a/test/api/runsDatastoreTest.js b/test/api/runsDatastoreTest.js new file mode 100644 index 0000000..1ac4d03 --- /dev/null +++ b/test/api/runsDatastoreTest.js @@ -0,0 +1,81 @@ +var should = require('chai').should(); +var runsDatastore = require('../../lib/server/datastores/runsDatastore'); + +describe('runsDatastore', function() { + + var datastore = new runsDatastore(); + + var firstRunId = 333; + var secondRunId = 999; + + it('should accept new runs', function() { + datastore.should.have.a.property('add').that.is.a('function'); + + datastore.add({ + runId: firstRunId, + otherData: 123456789 + }, 0); + + datastore.add({ + runId: secondRunId, + otherData: 'whatever' + }, 1); + }); + + it('should have stored the runs with a status "runnung"', function() { + datastore.should.have.a.property('get').that.is.a('function'); + + var firstRun = datastore.get(firstRunId); + firstRun.should.have.a.property('runId').that.equals(firstRunId); + firstRun.should.have.a.property('status').that.deep.equals({ + statusCode: 'running' + }); + + var secondRun = datastore.get(secondRunId); + secondRun.should.have.a.property('runId').that.equals(secondRunId); + secondRun.should.have.a.property('status').that.deep.equals({ + statusCode: 'awaiting', + position: 1 + }); + + }); + + it('should have exactly 2 runs in the store', function() { + var runs = datastore.list(); + runs.should.be.a('array'); + runs.should.have.length(2); + runs[0].should.have.a.property('runId').that.equals(firstRunId); + }); + + it('shoud update statuses correctly', function() { + + datastore.markAsComplete(firstRunId); + var firstRun = datastore.get(firstRunId); + firstRun.should.have.a.property('status').that.deep.equals({ + statusCode: 'complete' + }); + + datastore.updatePosition(secondRunId, 0); + var secondRun = datastore.get(secondRunId); + secondRun.should.have.a.property('status').that.deep.equals({ + statusCode: 'running' + }); + + datastore.markAsFailed(secondRunId); + secondRun = datastore.get(secondRunId); + secondRun.should.have.a.property('status').that.deep.equals({ + statusCode: 'failed' + }); + + }); + + it('should delete a run', function() { + datastore.delete(firstRunId); + + var runs = datastore.list(); + runs.should.be.a('array'); + runs.should.have.length(1); + + runs[0].should.have.a.property('runId').that.equals(secondRunId); + }); +}); diff --git a/test/server/runsQueueTest.js b/test/api/runsQueueTest.js similarity index 88% rename from test/server/runsQueueTest.js rename to test/api/runsQueueTest.js index 45a2a23..1e8798d 100644 --- a/test/server/runsQueueTest.js +++ b/test/api/runsQueueTest.js @@ -4,13 +4,15 @@ var runsQueue = require('../../lib/server/datastores/runsQueue.js'); describe('runsQueue', function() { var queue = new runsQueue(); + var aaaRun = null; + var bbbRun = null; var cccRun = null; it('should accept a new runId', function(done) { queue.should.have.a.property('push').that.is.a('function'); - var aaaRun = queue.push('aaa'); - queue.push('bbb'); + aaaRun = queue.push('aaa'); + bbbRun = queue.push('bbb'); aaaRun.then(function() { done(); @@ -20,9 +22,11 @@ describe('runsQueue', function() { it('should return the right positions', function() { var aaaPosition = queue.getPosition('aaa'); aaaPosition.should.equal(0); + aaaRun.startingPosition.should.equal(0); var bbbPosition = queue.getPosition('bbb'); bbbPosition.should.equal(1); + bbbRun.startingPosition.should.equal(1); var cccPosition = queue.getPosition('ccc'); cccPosition.should.equal(-1); diff --git a/test/api/phantomasWrapperTest.js b/test/core/phantomasWrapperTest.js similarity index 100% rename from test/api/phantomasWrapperTest.js rename to test/core/phantomasWrapperTest.js diff --git a/test/api/rulesCheckerTest.js b/test/core/rulesCheckerTest.js similarity index 100% rename from test/api/rulesCheckerTest.js rename to test/core/rulesCheckerTest.js diff --git a/test/api/yellowlabtoolsTest.js b/test/core/yellowlabtoolsTest.js similarity index 100% rename from test/api/yellowlabtoolsTest.js rename to test/core/yellowlabtoolsTest.js diff --git a/test/server/runsDatastoreTest.js b/test/server/runsDatastoreTest.js deleted file mode 100644 index 750956d..0000000 --- a/test/server/runsDatastoreTest.js +++ /dev/null @@ -1,41 +0,0 @@ -var should = require('chai').should(); -var runsDatastore = require('../../lib/server/datastores/runsDatastore'); - -describe('runsDatastore', function() { - - var datastore = new runsDatastore(); - - var randomId = Math.round(Math.random() * 100000); - - it('should accept a new run', function() { - datastore.should.have.a.property('add').that.is.a('function'); - - datastore.add({ - _id: randomId, - otherData: 123456789 - }); - }); - - it('should have stored the run', function() { - datastore.should.have.a.property('get').that.is.a('function'); - - var run = datastore.get(randomId); - - run.should.have.a.property('_id').that.equals(randomId); - }); - - it('should have exactly 1 run in the store', function() { - var runs = datastore.list(); - runs.should.be.a('array'); - runs.should.have.length(1); - runs[0].should.have.a.property('_id').that.equals(randomId); - }); - - it('should delete the run', function() { - datastore.delete(randomId); - - var runs = datastore.list(); - runs.should.be.a('array'); - runs.should.have.length(0); - }); -});