Split YLT into 2 projects. The server is now in the project YellowLabTools/YellowLabTools-server

This commit is contained in:
Gaël Métais
2021-04-24 02:40:30 +01:00
parent 97f779dc83
commit c9f63138a0
67 changed files with 6 additions and 6399 deletions
-681
View File
@@ -1,681 +0,0 @@
var should = require('chai').should();
var request = require('request');
var Q = require('q');
var config = {
"authorizedKeys": {
"1234567890": "contact@gaelmetais.com"
}
};
var serverUrl = 'http://localhost:8387';
var wwwUrl = 'http://localhost:8388';
describe('api', function() {
var syncRunResultUrl;
var asyncRunId;
var screenshotUrl;
it('should refuse a query with an invalid key', function(done) {
this.timeout(5000);
request({
method: 'POST',
url: serverUrl + '/api/runs',
body: {
url: wwwUrl + '/simple-page.html',
waitForResponse: false
},
json: true,
headers: {
'Content-Type': 'application/json',
'X-Api-Key': 'invalid'
}
}, function(error, response, body) {
if (!error && response.statusCode === 401) {
done();
} else {
done(error || response.statusCode);
}
});
});
it('should fail without an URL when asynchronous', function(done) {
this.timeout(15000);
request({
method: 'POST',
url: serverUrl + '/api/runs',
body: {
url: '',
waitForResponse: true
},
json: true,
headers: {
'Content-Type': 'application/json',
'X-Api-Key': Object.keys(config.authorizedKeys)[0]
}
}, function(error, response, body) {
if (!error && response.statusCode === 400) {
done();
} else {
done(error || response.statusCode);
}
});
});
it('should fail without an URL when synchronous', function(done) {
this.timeout(15000);
request({
method: 'POST',
url: serverUrl + '/api/runs',
body: {
url: '',
waitForResponse: true
},
json: true,
headers: {
'Content-Type': 'application/json',
'X-Api-Key': Object.keys(config.authorizedKeys)[0]
}
}, function(error, response, body) {
if (!error && response.statusCode === 400) {
done();
} else {
done(error || response.statusCode);
}
});
});
it('should launch a synchronous run', function(done) {
this.timeout(15000);
request({
method: 'POST',
url: serverUrl + '/api/runs',
body: {
url: wwwUrl + '/simple-page.html',
waitForResponse: true,
screenshot: true,
device: 'tablet',
//waitForSelector: '*',
cookie: 'foo=bar;domain=google.com',
authUser: 'joe',
authPass: 'secret'
},
json: true,
headers: {
'Content-Type': 'application/json',
'X-Api-Key': Object.keys(config.authorizedKeys)[0]
}
}, function(error, response, body) {
if (!error && response.statusCode === 302) {
response.headers.should.have.a.property('location').that.is.a('string');
syncRunResultUrl = response.headers.location;
done();
} else {
done(error || response.statusCode);
}
});
});
it('should return the rules only', function(done) {
this.timeout(15000);
request({
method: 'POST',
url: serverUrl + '/api/runs',
body: {
url: wwwUrl + '/simple-page.html',
waitForResponse: true,
partialResult: 'rules'
},
json: true,
headers: {
'Content-Type': 'application/json',
'X-Api-Key': Object.keys(config.authorizedKeys)[0]
}
}, function(error, response, body) {
if (!error && response.statusCode === 302) {
response.headers.should.have.a.property('location').that.is.a('string');
response.headers.location.should.contain('/rules');
done();
} else {
done(error || response.statusCode);
}
});
});
it('should retrieve the results for the synchronous run', function(done) {
this.timeout(15000);
request({
method: 'GET',
url: serverUrl + syncRunResultUrl,
json: true,
}, function(error, response, body) {
if (!error && response.statusCode === 200) {
body.should.have.a.property('runId').that.is.a('string');
body.should.have.a.property('params').that.is.an('object');
body.should.have.a.property('scoreProfiles').that.is.an('object');
body.should.have.a.property('rules').that.is.an('object');
body.should.have.a.property('toolsResults').that.is.an('object');
// Check if settings are correctly sent and retrieved
body.params.options.should.have.a.property('device').that.equals('tablet');
//body.params.options.should.have.a.property('waitForSelector').that.equals('*');
body.params.options.should.have.a.property('cookie').that.equals('foo=bar;domain=google.com');
body.params.options.should.have.a.property('authUser').that.equals('joe');
body.params.options.should.have.a.property('authPass').that.equals('secret');
// Check if the screenshot temporary file was correctly removed
body.params.options.should.not.have.a.property('screenshot');
// Check if the screenshot buffer was correctly removed
body.should.not.have.a.property('screenshotBuffer');
// Check if the screenshot url is here
body.should.have.a.property('screenshotUrl');
body.screenshotUrl.should.equal('api/results/' + body.runId + '/screenshot.jpg');
screenshotUrl = body.screenshotUrl;
done();
} else {
done(error || response.statusCode);
}
});
});
it('should launch a run without waiting for the response', function(done) {
this.timeout(5000);
request({
method: 'POST',
url: serverUrl + '/api/runs',
body: {
url: wwwUrl + '/simple-page.html',
waitForResponse: false,
jsTimeline: true
},
json: true,
headers: {
'Content-Type': 'application/json',
'X-Api-Key': Object.keys(config.authorizedKeys)[0]
}
}, function(error, response, body) {
if (!error && response.statusCode === 200) {
asyncRunId = body.runId;
asyncRunId.should.be.a('string');
done();
} else {
done(error || response.statusCode);
}
});
});
it('should respond run status: running', function(done) {
this.timeout(5000);
request({
method: 'GET',
url: serverUrl + '/api/runs/' + asyncRunId,
json: true,
headers: {
'Content-Type': 'application/json',
'X-Api-Key': Object.keys(config.authorizedKeys)[0]
}
}, function(error, response, body) {
if (!error && response.statusCode === 200) {
body.runId.should.equal(asyncRunId);
body.status.should.deep.equal({
statusCode: 'running'
});
done();
} else {
done(error || response.statusCode);
}
});
});
it('should accept up to 10 anonymous runs to the API', function(done) {
this.timeout(5000);
function launchRun() {
var deferred = Q.defer();
request({
method: 'POST',
url: serverUrl + '/api/runs',
body: {
url: wwwUrl + '/simple-page.html',
waitForResponse: false
},
json: true
}, function(error, response, body) {
lastRunId = body.runId;
if (error) {
deferred.reject(error);
} else {
deferred.resolve(response, body);
}
});
return deferred.promise;
}
launchRun()
.then(launchRun)
.then(launchRun)
.then(launchRun)
.then(launchRun)
.then(function(response, body) {
// Here should still be ok
response.statusCode.should.equal(200);
launchRun()
.then(launchRun)
.then(launchRun)
.then(launchRun)
.then(launchRun)
.then(launchRun)
.then(function(response, body) {
// It should fail now
response.statusCode.should.equal(429);
done();
})
.fail(function(error) {
done(error);
});
}).fail(function(error) {
done(error);
});
});
it('should respond 404 to unknown runId', function(done) {
this.timeout(5000);
request({
method: 'GET',
url: serverUrl + '/api/runs/unknown',
json: true
}, function(error, response, body) {
if (!error && response.statusCode === 404) {
done();
} else {
done(error || response.statusCode);
}
});
});
it('should respond 404 to unknown result', function(done) {
this.timeout(5000);
request({
method: 'GET',
url: serverUrl + '/api/results/unknown',
json: true
}, function(error, response, body) {
if (!error && response.statusCode === 404) {
done();
} else {
done(error || response.statusCode);
}
});
});
it('should respond status complete to the first run', function(done) {
this.timeout(12000);
function checkStatus() {
request({
method: 'GET',
url: serverUrl + '/api/runs/' + asyncRunId,
json: true
}, function(error, response, body) {
if (!error && response.statusCode === 200) {
body.runId.should.equal(asyncRunId);
if (body.status.statusCode === 'running') {
setTimeout(checkStatus, 250);
} else if (body.status.statusCode === 'complete') {
done();
} else {
done(body.status.statusCode);
}
} else {
done(error || response.statusCode);
}
});
}
checkStatus();
});
it('should find the result of the async run', function(done) {
this.timeout(5000);
request({
method: 'GET',
url: serverUrl + '/api/results/' + asyncRunId,
json: true,
}, function(error, response, body) {
if (!error && response.statusCode === 200) {
body.should.have.a.property('runId').that.equals(asyncRunId);
body.should.have.a.property('params').that.is.an('object');
body.params.url.should.equal(wwwUrl + '/simple-page.html');
body.should.have.a.property('scoreProfiles').that.is.an('object');
body.scoreProfiles.should.have.a.property('generic').that.is.an('object');
body.scoreProfiles.generic.should.have.a.property('globalScore').that.is.a('number');
body.scoreProfiles.generic.should.have.a.property('categories').that.is.an('object');
body.should.have.a.property('rules').that.is.an('object');
body.should.have.a.property('toolsResults').that.is.an('object');
body.toolsResults.should.have.a.property('phantomas').that.is.an('object');
done();
} else {
done(error || response.statusCode);
}
});
});
it('should return the generic score object', function(done) {
this.timeout(5000);
request({
method: 'GET',
url: serverUrl + '/api/results/' + asyncRunId + '/generalScores',
json: true,
}, function(error, response, body) {
if (!error && response.statusCode === 200) {
body.should.have.a.property('globalScore').that.is.a('number');
body.should.have.a.property('categories').that.is.an('object');
done();
} else {
done(error || response.statusCode);
}
});
});
it('should return the generic score object also', function(done) {
this.timeout(5000);
request({
method: 'GET',
url: serverUrl + '/api/results/' + asyncRunId + '/generalScores/generic',
json: true,
}, function(error, response, body) {
if (!error && response.statusCode === 200) {
body.should.have.a.property('globalScore').that.is.a('number');
body.should.have.a.property('categories').that.is.an('object');
done();
} else {
done(error || response.statusCode);
}
});
});
it('should not find an unknown score object', function(done) {
this.timeout(5000);
request({
method: 'GET',
url: serverUrl + '/api/results/' + asyncRunId + '/generalScores/unknown',
json: true,
}, function(error, response, body) {
if (!error && response.statusCode === 404) {
done();
} else {
done(error || response.statusCode);
}
});
});
it('should return the rules', function(done) {
this.timeout(5000);
request({
method: 'GET',
url: serverUrl + '/api/results/' + asyncRunId + '/rules',
json: true,
}, function(error, response, body) {
if (!error && response.statusCode === 200) {
var firstRule = body[Object.keys(body)[0]];
firstRule.should.have.a.property('policy').that.is.an('object');
firstRule.should.have.a.property('value').that.is.a('number');
firstRule.should.have.a.property('bad').that.is.a('boolean');
firstRule.should.have.a.property('abnormal').that.is.a('boolean');
firstRule.should.have.a.property('score').that.is.a('number');
firstRule.should.have.a.property('abnormalityScore').that.is.a('number');
done();
} else {
done(error || response.statusCode);
}
});
});
it('should return the phantomas results', function(done) {
this.timeout(5000);
request({
method: 'GET',
url: serverUrl + '/api/results/' + asyncRunId + '/toolsResults/phantomas',
json: true,
}, function(error, response, body) {
if (!error && response.statusCode === 200) {
body.should.have.a.property('metrics').that.is.an('object');
body.should.have.a.property('offenders').that.is.an('object');
done();
} else {
done(error || response.statusCode);
}
});
});
it('should return the entire object and exclude toolsResults', function(done) {
this.timeout(5000);
request({
method: 'GET',
url: serverUrl + '/api/results/' + asyncRunId + '?exclude=toolsResults',
json: true,
}, function(error, response, body) {
if (!error && response.statusCode === 200) {
body.should.have.a.property('runId').that.equals(asyncRunId);
body.should.have.a.property('params').that.is.an('object');
body.should.have.a.property('scoreProfiles').that.is.an('object');
body.should.have.a.property('rules').that.is.an('object');
body.should.not.have.a.property('toolsResults').that.is.an('object');
done();
} else {
done(error || response.statusCode);
}
});
});
it('should return the entire object and exclude params and toolsResults', function(done) {
this.timeout(5000);
request({
method: 'GET',
url: serverUrl + '/api/results/' + asyncRunId + '?exclude=toolsResults,params',
json: true,
}, function(error, response, body) {
if (!error && response.statusCode === 200) {
body.should.have.a.property('runId').that.equals(asyncRunId);
body.should.have.a.property('scoreProfiles').that.is.an('object');
body.should.have.a.property('rules').that.is.an('object');
body.should.not.have.a.property('params').that.is.an('object');
body.should.not.have.a.property('toolsResults').that.is.an('object');
done();
} else {
done(error || response.statusCode);
}
});
});
it('should return the entire object and don\'t exclude anything', function(done) {
this.timeout(5000);
request({
method: 'GET',
url: serverUrl + '/api/results/' + asyncRunId + '?exclude=',
json: true,
}, function(error, response, body) {
if (!error && response.statusCode === 200) {
body.should.have.a.property('runId').that.equals(asyncRunId);
body.should.have.a.property('scoreProfiles').that.is.an('object');
body.should.have.a.property('rules').that.is.an('object');
body.should.have.a.property('params').that.is.an('object');
body.should.have.a.property('toolsResults').that.is.an('object');
done();
} else {
done(error || response.statusCode);
}
});
});
it('should return the entire object and don\'t exclude anything', function(done) {
this.timeout(5000);
request({
method: 'GET',
url: serverUrl + '/api/results/' + asyncRunId + '?exclude=null',
json: true,
}, function(error, response, body) {
if (!error && response.statusCode === 200) {
body.should.have.a.property('runId').that.equals(asyncRunId);
body.should.have.a.property('scoreProfiles').that.is.an('object');
body.should.have.a.property('rules').that.is.an('object');
body.should.have.a.property('params').that.is.an('object');
body.should.have.a.property('toolsResults').that.is.an('object');
done();
} else {
done(error || response.statusCode);
}
});
});
it('should retrieve the screenshot', function(done) {
this.timeout(5000);
request({
method: 'GET',
url: serverUrl + '/' + screenshotUrl
}, function(error, response, body) {
if (!error && response.statusCode === 200) {
response.headers['content-type'].should.equal('image/jpeg');
done();
} else {
done(error || response.statusCode);
}
});
});
it('should fail on a unexistant screenshot', function(done) {
this.timeout(5000);
request({
method: 'GET',
url: serverUrl + '/api/results/000000/screenshot.jpg'
}, function(error, response, body) {
if (!error && response.statusCode === 404) {
done();
} else {
done(error || response.statusCode);
}
});
});
it('should refuse a query on a blocked Url', function(done) {
this.timeout(5000);
request({
method: 'POST',
url: serverUrl + '/api/runs',
body: {
url: 'http://www.test.com/something.html',
waitForResponse: false
},
json: true,
headers: {
'Content-Type': 'application/json',
'X-Api-Key': Object.keys(config.authorizedKeys)[0]
}
}, function(error, response, body) {
if (!error && response.statusCode === 403) {
done();
} else {
done(error || response.statusCode);
}
});
});
});
-121
View File
@@ -1,121 +0,0 @@
var should = require('chai').should();
var resultsDatastore = require('../../lib/server/datastores/resultsDatastore');
var fs = require('fs');
var path = require('path');
describe('resultsDatastore', function() {
var datastore = new resultsDatastore();
var testId1 = '123456789';
var testData1 = {
runId: testId1,
other: {
foo: 'foo',
bar: 1
}
};
it('should store a result', function(done) {
datastore.should.have.a.property('saveResult').that.is.a('function');
datastore.saveResult(testData1).then(function() {
done();
}).fail(function(err) {
done(err);
});
});
it('should store another result', function(done) {
var testData2 = {
runId: '987654321',
other: {
foo: 'foo',
bar: 2
}
};
datastore.saveResult(testData2).then(function() {
done();
}).fail(function(err) {
done(err);
});
});
it('should retrieve a result', function(done) {
datastore.getResult(testId1)
.then(function(results) {
// Compare results with testData
results.should.deep.equal(testData1);
done();
}).fail(function(err) {
done(err);
});
});
it('should delete a result', function(done) {
datastore.deleteResult(testId1)
.then(function() {
done();
}).fail(function(err) {
done(err);
});
});
it('should not find the result anymore', function(done) {
datastore.getResult(testId1)
.then(function(results) {
done('Error, the result is still in the datastore');
}).fail(function(err) {
done();
});
});
var testId3 = '555555';
var testData3 = {
runId: testId3,
other: {
foo: 'foo',
bar: 2
},
screenshotBuffer: fs.readFileSync(path.join(__dirname, '../fixtures/logo-large.png'))
};
it('should store a test with a screenshot', function(done) {
datastore.saveResult(testData3).then(function() {
done();
}).fail(function(err) {
done(err);
});
});
it('should have a normal result', function(done) {
datastore.getResult(testId3)
.then(function(results) {
results.should.not.have.a.property('screenshot');
done();
})
.fail(function(err) {
done(err);
});
});
it('should retrieve the saved image', function() {
datastore.getScreenshot(testId3)
.then(function(imageBuffer) {
imageBuffer.should.be.an.instanceof(Buffer);
done();
})
.fail(function(err) {
done(err);
});
});
});
-82
View File
@@ -1,82 +0,0 @@
var should = require('chai').should();
var runsDatastore = require('../../lib/server/datastores/runsDatastore');
describe('runsDatastore', function() {
var datastore = new runsDatastore();
var firstRunId = 333;
var secondRunId = 999;
it('should accept new runs', function() {
datastore.should.have.a.property('add').that.is.a('function');
datastore.add({
runId: firstRunId,
otherData: 123456789
}, 0);
datastore.add({
runId: secondRunId,
otherData: 'whatever'
}, 1);
});
it('should have stored the runs with a status "runnung"', function() {
datastore.should.have.a.property('get').that.is.a('function');
var firstRun = datastore.get(firstRunId);
firstRun.should.have.a.property('runId').that.equals(firstRunId);
firstRun.should.have.a.property('status').that.deep.equals({
statusCode: 'running'
});
var secondRun = datastore.get(secondRunId);
secondRun.should.have.a.property('runId').that.equals(secondRunId);
secondRun.should.have.a.property('status').that.deep.equals({
statusCode: 'awaiting',
position: 1
});
});
it('should have exactly 2 runs in the store', function() {
var runs = datastore.list();
runs.should.be.a('array');
runs.should.have.length(2);
runs[0].should.have.a.property('runId').that.equals(firstRunId);
});
it('shoud update statuses correctly', function() {
datastore.markAsComplete(firstRunId);
var firstRun = datastore.get(firstRunId);
firstRun.should.have.a.property('status').that.deep.equals({
statusCode: 'complete'
});
datastore.updatePosition(secondRunId, 0);
var secondRun = datastore.get(secondRunId);
secondRun.should.have.a.property('status').that.deep.equals({
statusCode: 'running'
});
datastore.markAsFailed(secondRunId, 'Error message');
secondRun = datastore.get(secondRunId);
secondRun.should.have.a.property('status').that.deep.equals({
statusCode: 'failed',
error: 'Error message'
});
});
it('should delete a run', function() {
datastore.delete(firstRunId);
var runs = datastore.list();
runs.should.be.a('array');
runs.should.have.length(1);
runs[0].should.have.a.property('runId').that.equals(secondRunId);
});
});
-68
View File
@@ -1,68 +0,0 @@
var should = require('chai').should();
var runsQueue = require('../../lib/server/datastores/runsQueue');
describe('runsQueue', function() {
var queue = new runsQueue();
var aaaRun = null;
var bbbRun = null;
var cccRun = null;
it('should accept a new runId', function(done) {
queue.should.have.a.property('push').that.is.a('function');
aaaRun = queue.push('aaa');
bbbRun = queue.push('bbb');
aaaRun.then(function() {
done();
});
});
it('should return the right positions', function() {
var aaaPosition = queue.getPosition('aaa');
aaaPosition.should.equal(0);
aaaRun.startingPosition.should.equal(0);
var bbbPosition = queue.getPosition('bbb');
bbbPosition.should.equal(1);
bbbRun.startingPosition.should.equal(1);
var cccPosition = queue.getPosition('ccc');
cccPosition.should.equal(-1);
});
it('should refresh runs\' positions', function(done) {
cccRun = queue.push('ccc');
cccRun.progress(function(position) {
position.should.equal(1);
var positionDoubleCheck = queue.getPosition('ccc');
positionDoubleCheck.should.equal(1);
done();
});
queue.remove('aaa');
});
it('should fulfill the promise when first in the line', function(done) {
cccRun.then(function() {
done();
});
queue.remove('bbb');
});
it('should not keep removed runs', function() {
var aaaPosition = queue.getPosition('aaa');
aaaPosition.should.equal(-1);
var bbbPosition = queue.getPosition('bbb');
bbbPosition.should.equal(-1);
var cccPosition = queue.getPosition('ccc');
cccPosition.should.equal(0);
});
});
-78
View File
@@ -1,78 +0,0 @@
var should = require('chai').should();
var ScreenshotHandler = require('../../lib/screenshotHandler');
var fs = require('fs');
var path = require('path');
var rimraf = require('rimraf');
describe('screenshotHandler', function() {
var imagePath = path.join(__dirname, '../fixtures/logo-large.png');
var screenshot, jimpImage;
it('should open an image and return an jimp object', function(done) {
ScreenshotHandler.openImage(imagePath)
.then(function(image) {
jimpImage = image;
jimpImage.should.be.an('object');
jimpImage.bitmap.width.should.equal(620);
jimpImage.bitmap.height.should.equal(104);
done();
})
.fail(function(err) {
done(err);
});
});
it('should resize an jimp image', function(done) {
ScreenshotHandler.resizeImage(jimpImage, 310)
.then(function(image) {
jimpImage = image;
jimpImage.bitmap.width.should.equal(310);
jimpImage.bitmap.height.should.equal(52);
done();
})
.fail(function(err) {
done(err);
});
});
it('should transform a jimp image into a buffer', function(done) {
ScreenshotHandler.toBuffer(jimpImage)
.then(function(buffer) {
buffer.should.be.an.instanceof(Buffer);
done();
})
.fail(function(err) {
done(err);
});
});
it('should create the tmp folder if it doesn\'t exist', function(done) {
// Delete tmp folder if it exists
rimraf.sync("/some/directory");
// The function we want to test
ScreenshotHandler.createTmpScreenshotFolder()
.then(function(buffer) {
fs.existsSync(path.join(__dirname, '../../tmp')).should.equal(true);
done();
})
.fail(function(err) {
done(err);
});
});
it('should return the tmp folder path', function() {
ScreenshotHandler.getTmpFileRelativePath().should.equal('tmp/temp-screenshot.png');
});
});
-8
View File
@@ -1,8 +0,0 @@
# Stress tests with Gatling
[Gatling](http://gatling.io/) is an open source stress test tool
Objective: 500 simultaneous users on the website
The YLTWebInterfaceSimulation.scala file is a scenario simulating a user coming on the web page and launching a run.
@@ -1,42 +0,0 @@
package computerdatabase // 1
import io.gatling.core.Predef._ // 2
import io.gatling.http.Predef._
import scala.concurrent.duration._
class YLTWebInterfaceSimulation extends Simulation {
val httpConf = http
.baseURL("http://localhost:8383")
.acceptHeader("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
.doNotTrackHeader("1")
.acceptLanguageHeader("en-US,en;q=0.5")
.acceptEncodingHeader("gzip, deflate")
.userAgentHeader("Mozilla/5.0 (Windows NT 5.1; rv:31.0) Gecko/20100101 Firefox/31.0")
val scn = scenario("YLTWebInterfaceSimulation")
.exec(http("home page")
.get("/")
)
.exec(http("static asset")
.get("/front/fonts/icons.woff")
)
.pause(100 milliseconds)
.exec(http("launch run")
.post("/api/runs")
.body(StringBody("""{ "url": "http://www.google.com", "waitForResponse":false }""")).asJSON
)
.repeat(10, "loop") {
exec(http("get status")
.get("/api/runs/dzlqsahu8d")
)
.pause(2000 milliseconds)
}
.exec(http("get result")
.get("/api/results/dzlqsahu8d")
)
setUp(
scn.inject(rampUsers(1000) over(60 seconds))
).protocols(httpConf)
}