WEBDIENSTE/YellowLabTools
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

API is starting to run

Browse Source
This commit is contained in:
Gaël Métais
2014-12-05 16:14:08 +01:00
parent e18cd87a07
commit 906f0876a0
18 changed files with 423 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Gruntfile.js
+2 -2
View File
@@ -71,13 +71,13 @@ module.exports = function(grunt) {
options: {
reporter: 'spec',
},
src: ['coverage/test/api/*.js', 'coverage/test/server/*.js']
src: ['coverage/test/core/*.js', 'coverage/test/api/*.js']
},
'test-current-work': {
options: {
reporter: 'spec',
},
src: ['coverage/test/server/runsQueueTest.js']
src: ['coverage/test/server/runsDatastoreTest.js']
},
coverage: {
options: {
bin/server.js
+1 -1
View File
@@ -8,7 +8,7 @@ var bodyParser = require('body-parser');
var compress = require('compression');
app.use(compress());
app.use(bodyParser.urlencoded({ extended: false }));
app.use(bodyParser.json());
// Initialize the controllers
lib/metadata/policies.json
+1 -1
View File
@@ -365,6 +365,6 @@
"message": "<p>For each domain met, the browser needs to make a DNS look-up, which is slow. Avoid having to many different domains and the page should render faster.</p><p>By the way, domain sharding is not a good practice anymore.</p>",
"isOkThreshold": 10,
"isBadThreshold": 25,
"isAbnormalThreshold": 40
"isAbnormalThreshold": 50
}
}
lib/server/controllers/apiController.js
+110 -44
View File
@@ -1,86 +1,152 @@
var debug = require('debug')('ylt:server');
var debug = require('debug')('ylt:server');
var runsQueue = require('../datastores/runsQueue');
var runsDatastore = require('../datastores/runsDatastore');
var YellowLabTools = require('../../yellowlabtools');
var RunsQueue = require('../datastores/runsQueue');
var RunsDatastore = require('../datastores/runsDatastore');
var ResultsDatastore = require('../datastores/resultsDatastore');
function ApiController(app) {
var ApiController = function(app) {
'use strict';
var queue = new RunsQueue();
var runsDatastore = new RunsDatastore();
var resultsDatastore = new ResultsDatastore();
// Retrieve the list of all runs
/*app.get('/runs', function(req, res) {
/*app.get('/api/runs', function(req, res) {
// NOT YET
});*/
// Create a new run
app.post('/runs', function(req, res) {
app.post('/api/runs', function(req, res) {
// Grab the test parameters
// Grab the test parameters and generate a random run ID
var run = {
// Generate a random run ID
_id: (Date.now()*1000 + Math.round(Math.random()*1000)).toString(36),
runId: (Date.now()*1000 + Math.round(Math.random()*1000)).toString(36),
params: {
url: req.body.url,
waitForResponse: req.body.waitForResponse || true
waitForResponse: req.body.waitForResponse !== false
}
};
// Add test to the testQueue
debug('Adding test %s to the queue', run._id);
var queuing = runsQueue.push(run._id);
debug('Adding test %s to the queue', run.runId);
var queuePromise = queue.push(run.runId);
// Save the run to the datastore
var position = runsQueue.getPosition(run._id);
run.status = {
statusCode: (position === 0) ? STATUS_RUNNING : STATUS_AWAITING,
position: position
};
runsDatastore.add(run);
runsDatastore.add(run, queuePromise.startingPosition);
// Listening for position updates
queuing.progress(function(position) {
var savedRun = runsDatastore.get(run._id);
savedRun.status = {
statusCode: STATUS_AWAITING,
position: position
};
runsDatastore.update(savedRun);
queuePromise.progress(function(position) {
runsDatastore.updatePosition(run.runId, position);
});
// Let's start the run
queuePromise.then(function() {
queuing.then(function() {
runsDatastore.updatePosition(run.runId, 0);
debug('Launching test %s on %s', run.runId, run.params.url);
new YellowLabTools(run.params.url)
.then(function(data) {
debug('Success');
runsDatastore.markAsComplete(run.runId);
// Save result in datastore
data.runId = run.runId;
resultsDatastore.saveResult(data);
// Send result if the user was waiting
if (run.params.waitForResponse) {
res.redirect(302, '/api/results/' + run.runId);
}
}).fail(function(err) {
console.error('Test failed for %s', run.params.url);
console.error(err);
console.error(err.stack);
runsDatastore.markAsFailed(run.runId);
}).finally(function() {
queue.remove(run.runId);
});
}).fail(function(err) {
console.error('Error or YLT\'s core instanciation');
console.error(err);
console.error(err.stack);
});
// The user doesn't not want to wait for the response
if (!params.waitForResponse) {
// Sending just the test id
res.setHeader('Content-Type', 'application/javascript');
res.send(JSON.stringify({
testId: testId
}));
// The user doesn't not want to wait for the response, sending the run ID only
if (!run.params.waitForResponse) {
console.log('Sending response without waiting.');
res.setHeader('Content-Type', 'application/json');
res.send(JSON.stringify({runId: run.runId}));
}
});
// Retrive one run by id
app.get('/run/:id', function(req, res) {
app.get('/api/runs/:id', function(req, res) {
var runId = req.params.id;
var run = runsDatastore.get(runId);
if (run) {
res.setHeader('Content-Type', 'application/json');
res.send(JSON.stringify(run, null, 2));
} else {
res.status(404).send('Not found');
}
});
// Delete one run by id
/*app.delete('/run/:id', function(req, res) {
// NOT YET
/*app.delete('/api/runs/:id', function(req, res) {
deleteRun()
});*/
// Delete all
/*app.delete('/api/runs', function(req, res) {
purgeRuns()
});
var STATUS_AWAITING = 'awaiting';
var STATUS_RUNNING = 'running';
var STATUS_DONE = 'done';
var STATUS_FAILED = 'failed';
// List all
app.get('/api/runs', function(req, res) {
listRuns()
});
}
// Exists
app.head('/api/runs/:id', function(req, res) {
existsX();
// Retourne 200 si existe ou 404 si n'existe pas
});
*/
// Retrive one result by id
app.get('/api/results/:id', function(req, res) {
var runId = req.params.id;
resultsDatastore.getResult(runId)
.then(function(data) {
// This is the pivot format, we might need to clean it first?
// Hide phantomas results
data.toolsResults.phantomas = {};
res.setHeader('Content-Type', 'application/json');
res.send(JSON.stringify(data, null, 2));
}).fail(function() {
res.status(404).send('Not found');
});
});
};
module.exports = ApiController;
lib/server/controllers/uiController.js
+13
View File
@@ -0,0 +1,13 @@
var UiController = function(app) {
'use strict';
// Create a new run
app.get('/', function(req, res) {
res.setHeader('Content-Type', 'text/html');
res.send('Test');
});
};
module.exports = UiController;
lib/server/datastores/resultsDatastore.js
+71 -1
View File
@@ -1,9 +1,79 @@
var fs = require('fs');
var rimraf = require('rimraf');
var path = require('path');
var Q = require('q');
function ResultsDatastore() {
'use strict';
var resultFileName = 'results.json';
var resultsFolderName = 'results';
var resultsDir = path.join(__dirname, '..', '..', '..', resultsFolderName);
this.saveResult = function(testResults) {
var promise = createResultFolder(testResults.runId);
promise.then(function() {
var resultFilePath = path.join(resultsDir, testResults.runId, resultFileName);
return Q.nfcall(fs.writeFile, resultFilePath, JSON.stringify(testResults, null, 2));
});
return promise;
};
this.getResult = function(runId) {
var resultFilePath = path.join(resultsDir, runId, resultFileName);
return Q.nfcall(fs.readFile, resultFilePath, {encoding: 'utf8'}).then(function(data) {
return JSON.parse(data);
});
};
this.deleteResult = function(runId) {
var folder = path.join(resultsDir, runId);
return Q.nfcall(rimraf, folder);
};
// The folder /results/folderName/
function createResultFolder(folderName) {
var folder = path.join(resultsDir, folderName);
return createGlobalFolder().then(function() {
return Q.nfcall(fs.mkdir, folder);
});
}
// The folder /results/
function createGlobalFolder() {
var deferred = Q.defer();
// Create the results folder if it doesn't exist
fs.exists(resultsDir, function(exists) {
if (exists) {
deferred.resolve();
} else {
fs.mkdir(resultsDir, function(err) {
if (err) {
deferred.reject(err);
} else {
deferred.resolve();
}
});
}
});
return deferred.promise;
}
}
module.exports = ResultsDatastore;
lib/server/datastores/runsDatastore.js
+49 -4
View File
@@ -7,23 +7,68 @@ function RunsDatastore() {
// For the moment, maybe one day
var runs = {};
var STATUS_AWAITING = 'awaiting';
var STATUS_RUNNING = 'running';
var STATUS_COMPLETE = 'complete';
var STATUS_FAILED = 'failed';
this.add = function(run) {
runs[run._id] = run;
this.add = function(run, position) {
runs[run.runId] = run;
this.updatePosition(run.runId, position);
};
this.get = function(runId) {
return runs[runId];
};
this.update = function(run) {
runs[run._id] = run;
this.updatePosition = function(runId, position) {
var run = runs[runId];
if (position > 0) {
run.status = {
statusCode: STATUS_AWAITING,
position: position
};
} else {
run.status = {
statusCode: STATUS_RUNNING
};
}
runs[runId] = run;
};
this.markAsComplete = function(runId) {
var run = runs[runId];
run.status = {
statusCode: STATUS_COMPLETE
};
runs[runId] = run;
};
this.markAsFailed = function(runId) {
var run = runs[runId];
run.status = {
statusCode: STATUS_FAILED
};
runs[runId] = run;
};
this.delete = function(runId) {
delete runs[runId];
};
this.list = function() {
var runsArray = [];
Object.keys(runs).forEach(function(key) {
lib/server/datastores/runsQueue.js
+9 -2
View File
@@ -9,8 +9,9 @@ function RunsQueue() {
this.push = function(runId) {
var deferred = Q.defer();
var startingPosition = queue.length;
if (queue.length === 0) {
if (startingPosition === 0) {
// The queue is empty, let's run immediatly
queue.push({
@@ -32,7 +33,9 @@ function RunsQueue() {
});
}
return deferred.promise;
var promise = deferred.promise;
promise.startingPosition = startingPosition;
return promise;
};
@@ -68,6 +71,10 @@ function RunsQueue() {
});
};
this.length = function() {
return queue.length;
};
}
module.exports = RunsQueue;
lib/tools/phantomasWrapper.js
+1 -1
View File
@@ -64,7 +64,7 @@ var PhantomasWrapper = function() {
}, 600000);
// It's time to launch the test!!!
var triesNumber = 3;
var triesNumber = 2;
async.retry(triesNumber, function(cb) {
phantomas(task.url, options, function(err, json, results) {
lib/yellowlabtools.js
+4 -9
View File
@@ -14,24 +14,19 @@ var YellowLabTools = function(url, options) {
} else {
// Generate a test id
var testId = (Date.now()*1000 + Math.round(Math.random()*1000)).toString(36);
if (url.indexOf('http://') !== 0 && url.indexOf('https://') !== 0) {
url = 'http://' + url;
}
var params = {
testId: testId,
url: url,
options: options || {}
};
var runner = new Runner(params);
runner.then(function(data) {
deferred.resolve(data);
});
var runner = new Runner(params)
.then(function(data) {
deferred.resolve(data);
});
}
return deferred.promise;
package.json
+1
View File
@@ -16,6 +16,7 @@
"debug": "^2.1.0",
"express": "~4.10.4",
"phantomas": "1.7.0",
"rimraf": "^2.2.8",
"socket.io": "~1.2.0"
},
"devDependencies": {
test/api/resultsDatastoreTest.js
+74
View File
@@ -0,0 +1,74 @@
var should = require('chai').should();
var resultsDatastore = require('../../lib/server/datastores/resultsDatastore');
describe('resultsDatastore', function() {
var datastore = new resultsDatastore();
var testId1 = '123456789';
var testData1 = {
runId: testId1,
other: {
foo: 'foo',
bar: 1
}
};
it('should store a result', function(done) {
datastore.should.have.a.property('saveResult').that.is.a('function');
datastore.saveResult(testData1).then(function() {
done();
}).fail(function(err) {
done(err);
});
});
it('should store another result', function(done) {
var testData2 = {
runId: '987654321',
other: {
foo: 'foo',
bar: 2
}
};
datastore.saveResult(testData2).then(function() {
done();
}).fail(function(err) {
done(err);
});
});
it('should retrieve a result', function(done) {
datastore.getResult(testId1)
.then(function(results) {
// Compare results with testData
results.should.deep.equal(testData1);
done();
}).fail(function(err) {
done(err);
});
});
it('should delete a result', function(done) {
datastore.deleteResult(testId1)
.then(function() {
done();
}).fail(function(err) {
done(err);
});
});
it('should not find the result anymore', function(done) {
datastore.getResult(testId1)
.then(function(results) {
done('Error, the result is still in the datastore');
}).fail(function(err) {
done();
});
});
});
test/api/runsDatastoreTest.js
+81
View File
@@ -0,0 +1,81 @@
var should = require('chai').should();
var runsDatastore = require('../../lib/server/datastores/runsDatastore');
describe('runsDatastore', function() {
var datastore = new runsDatastore();
var firstRunId = 333;
var secondRunId = 999;
it('should accept new runs', function() {
datastore.should.have.a.property('add').that.is.a('function');
datastore.add({
runId: firstRunId,
otherData: 123456789
}, 0);
datastore.add({
runId: secondRunId,
otherData: 'whatever'
}, 1);
});
it('should have stored the runs with a status "runnung"', function() {
datastore.should.have.a.property('get').that.is.a('function');
var firstRun = datastore.get(firstRunId);
firstRun.should.have.a.property('runId').that.equals(firstRunId);
firstRun.should.have.a.property('status').that.deep.equals({
statusCode: 'running'
});
var secondRun = datastore.get(secondRunId);
secondRun.should.have.a.property('runId').that.equals(secondRunId);
secondRun.should.have.a.property('status').that.deep.equals({
statusCode: 'awaiting',
position: 1
});
});
it('should have exactly 2 runs in the store', function() {
var runs = datastore.list();
runs.should.be.a('array');
runs.should.have.length(2);
runs[0].should.have.a.property('runId').that.equals(firstRunId);
});
it('shoud update statuses correctly', function() {
datastore.markAsComplete(firstRunId);
var firstRun = datastore.get(firstRunId);
firstRun.should.have.a.property('status').that.deep.equals({
statusCode: 'complete'
});
datastore.updatePosition(secondRunId, 0);
var secondRun = datastore.get(secondRunId);
secondRun.should.have.a.property('status').that.deep.equals({
statusCode: 'running'
});
datastore.markAsFailed(secondRunId);
secondRun = datastore.get(secondRunId);
secondRun.should.have.a.property('status').that.deep.equals({
statusCode: 'failed'
});
});
it('should delete a run', function() {
datastore.delete(firstRunId);
var runs = datastore.list();
runs.should.be.a('array');
runs.should.have.length(1);
runs[0].should.have.a.property('runId').that.equals(secondRunId);
});
});
test/server/runsQueueTest.js → test/api/runsQueueTest.js
+6 -2
View File
@@ -4,13 +4,15 @@ var runsQueue = require('../../lib/server/datastores/runsQueue.js');
describe('runsQueue', function() {
var queue = new runsQueue();
var aaaRun = null;
var bbbRun = null;
var cccRun = null;
it('should accept a new runId', function(done) {
queue.should.have.a.property('push').that.is.a('function');
var aaaRun = queue.push('aaa');
queue.push('bbb');
aaaRun = queue.push('aaa');
bbbRun = queue.push('bbb');
aaaRun.then(function() {
done();
@@ -20,9 +22,11 @@ describe('runsQueue', function() {
it('should return the right positions', function() {
var aaaPosition = queue.getPosition('aaa');
aaaPosition.should.equal(0);
aaaRun.startingPosition.should.equal(0);
var bbbPosition = queue.getPosition('bbb');
bbbPosition.should.equal(1);
bbbRun.startingPosition.should.equal(1);
var cccPosition = queue.getPosition('ccc');
cccPosition.should.equal(-1);
test/api/phantomasWrapperTest.js → test/core/phantomasWrapperTest.js
View File
test/api/rulesCheckerTest.js → test/core/rulesCheckerTest.js
View File
test/api/yellowlabtoolsTest.js → test/core/yellowlabtoolsTest.js
View File
test/server/runsDatastoreTest.js
-41
View File
@@ -1,41 +0,0 @@
var should = require('chai').should();
var runsDatastore = require('../../lib/server/datastores/runsDatastore');
describe('runsDatastore', function() {
var datastore = new runsDatastore();
var randomId = Math.round(Math.random() * 100000);
it('should accept a new run', function() {
datastore.should.have.a.property('add').that.is.a('function');
datastore.add({
_id: randomId,
otherData: 123456789
});
});
it('should have stored the run', function() {
datastore.should.have.a.property('get').that.is.a('function');
var run = datastore.get(randomId);
run.should.have.a.property('_id').that.equals(randomId);
});
it('should have exactly 1 run in the store', function() {
var runs = datastore.list();
runs.should.be.a('array');
runs.should.have.length(1);
runs[0].should.have.a.property('_id').that.equals(randomId);
});
it('should delete the run', function() {
datastore.delete(randomId);
var runs = datastore.list();
runs.should.be.a('array');
runs.should.have.length(0);
});
});