WEBDIENSTE/YellowLabTools
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Two version of the API: normal/lambda. Will be merged later

Browse Source
This commit is contained in:
Gaël Métais
2021-03-20 18:24:11 +00:00
parent ec7cba8f49
commit 47eab96a67
3 changed files with 433 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bin/server.js
+4 -5
View File
@@ -25,15 +25,14 @@ app.engine('.html', require('ejs').__express);
app.set('view engine', 'ejs');
// Initialize the controllers
var apiController = require('../lib/server/controllers/apiController')(app);
var frontController = require('../lib/server/controllers/frontController')(app);
// Let's start the server!
if (!process.env.GRUNTED) {
var settings = require('../server_config/settings.json');
// Initialize the controllers
var apiController = settings.awsHosting ? require('../lib/server/controllers/awsApiController')(app) : require('../lib/server/controllers/apiController')(app);
var frontController = require('../lib/server/controllers/frontController')(app);
app.locals.baseUrl = settings.baseUrl;
server.listen(settings.serverPort, function() {
lib/server/controllers/apiController.js
+119 -66
View File
@@ -1,16 +1,14 @@
var debug = require('debug')('ylt:server');
var Q = require('q');
var AWS = require('aws-sdk');
var ylt = require('../../index');
var ScreenshotHandler = require('../../screenshotHandler');
var RunsQueue = require('../datastores/runsQueue');
var RunsDatastore = require('../datastores/runsDatastore');
var ResultsDatastore = require('../datastores/resultsDatastore');
var serverSettings = (process.env.IS_TEST) ? require('../../../test/fixtures/settings.json') : require('../../../server_config/settings.json');
var ResultsDatastore = (serverSettings.awsHosting) ? require('../datastores/awsResultsDatastore') : require('../datastores/resultsDatastore');
var ApiController = function(app) {
'use strict';
@@ -53,13 +51,21 @@ var ApiController = function(app) {
}
};
// Create the tmp folder if it doesn't exist
ScreenshotHandler.createTmpScreenshotFolder(run.runId);
// Add test to the testQueue
debug('Adding test %s to the queue', run.runId);
var queuePromise = queue.push(run.runId);
// Save the run to the datastore
//runsDatastore.add(run, queuePromise.startingPosition);
runsDatastore.add(run, 0);
runsDatastore.add(run, queuePromise.startingPosition);
// Listening for position updates
queuePromise.progress(function(position) {
runsDatastore.updatePosition(run.runId, position);
});
// Let's start the run
queuePromise.then(function() {
@@ -81,82 +87,129 @@ var ApiController = function(app) {
noExternals: run.params.noExternals
};
const {region, arn} = chooseLambdaRegionByGeoIP(req.headers);
const lambda = new AWS.Lambda({region: region});
return ylt(run.params.url, runOptions)
// Update the progress bar on each progress
.progress(function(progress) {
runsDatastore.updateRunProgress(run.runId, progress);
});
})
// Phantomas completed
.then(function(data) {
debug('Success');
data.runId = run.runId;
return lambda.invoke({
FunctionName: arn,
InvocationType: 'RequestResponse',
Payload: JSON.stringify({url: run.params.url, id: run.runId, options: runOptions})
}).promise();
// Some conditional steps exist if there is a screenshot
var screenshotPromise = Q.resolve();
})
if (run.params.screenshot) {
.then(function(response) {
debug('We\'ve got a response from AWS Lambda');
debug('StatusCode = %d', response.StatusCode);
debug('Payload = %s', response.Payload);
var screenshotSize = serverSettings.screenshotWidth ? serverSettings.screenshotWidth[run.params.device] : 400;
// Replace the empty promise created earlier with Q.resolve()
screenshotPromise = ScreenshotHandler.findAndOptimizeScreenshot(screenshotSize)
// Read screenshot
.then(function(screenshotBuffer) {
if (screenshotBuffer) {
debug('Image optimized');
data.screenshotBuffer = screenshotBuffer;
data.screenshotUrl = '/api/results/' + data.runId + '/screenshot.jpg';
}
})
// Don't worry if there's an error
.fail(function(err) {
debug('An error occured while creating the screenshot\'s thumbnail. Ignoring and continuing...');
debug(err);
});
if (response.StatusCode === 200 && response.Payload && response.Payload !== 'null') {
debug('Success!');
runsDatastore.markAsComplete(run.runId);
} else {
debug('Empty response from the lambda agent');
runsDatastore.markAsFailed(run.runId, "Empty response from the agent");
}
// Let's continue
return screenshotPromise
// Save results
.then(function() {
// Remove uneeded temp screenshot path
delete data.params.options.screenshot;
// Here we can remove tools results if not needed
delete data.toolsResults.phantomas.offenders.requests;
return resultsDatastore.saveResult(data);
})
// Mark as the run as complete and send the response if the request is still waiting
.then(function() {
debug('Result saved in datastore');
runsDatastore.markAsComplete(run.runId);
if (run.params.waitForResponse) {
// If the user only wants a portion of the result (partialResult option)
switch(run.params.partialResult) {
case 'generalScores':
res.redirect(302, '/api/results/' + run.runId + '/generalScores');
break;
case 'rules':
res.redirect(302, '/api/results/' + run.runId + '/rules');
break;
case 'javascriptExecutionTree':
res.redirect(302, '/api/results/' + run.runId + '/javascriptExecutionTree');
break;
case 'phantomas':
res.redirect(302, '/api/results/' + run.runId + '/toolsResults/phantomas');
break;
default:
res.redirect(302, '/api/results/' + run.runId);
}
}
})
.fail(function(err) {
console.error('Test failed for URL: %s', run.params.url);
console.error(err.toString());
runsDatastore.markAsFailed(run.runId, err.toString());
res.status(500).send('An error occured');
});
})
.catch(function(err) {
debug('Error from AWS Lambda:');
debug(err);
.fail(function(err) {
console.error('Test failed for URL: %s', run.params.url);
console.error(err.toString());
runsDatastore.markAsFailed(run.runId, err.toString());
res.status(400).send('Bad request');
})
.finally(function() {
queue.remove(run.runId);
});
// The user doesn't want to wait for the response, sending the run ID only
debug('Sending response without waiting.');
res.setHeader('Content-Type', 'application/json');
res.send(JSON.stringify({runId: run.runId}));
if (!run.params.waitForResponse) {
debug('Sending response without waiting.');
res.setHeader('Content-Type', 'application/json');
res.send(JSON.stringify({runId: run.runId}));
}
});
// Reads the Geoip_Continent_Code header and chooses the right region from the settings
function chooseLambdaRegionByGeoIP(headers) {
// The settings can be configured like this in server_config/settings.json:
//
// "awsHosting": {
// "lambda": {
// "regionByContinent": {
// "AF": "eu-west-3",
// "AS": "ap-southeast-1",
// "EU": "eu-west-3",
// "NA": "us-east-1",
// "OC": "ap-southeast-1",
// "SA": "us-east-1",
// "default": "eu-west-3"
// },
// "arnByRegion": {
// "us-east-1": "arn:aws:lambda:us-east-1:xxx:function:xxx",
// "eu-west-3": "arn:aws:lambda:eu-west-3:xxx:function:xxx",
// "ap-southeast-1": "arn:aws:lambda:ap-southeast-1:xxx:function:xxx"
// }
// }
// },
const header = headers.geoip_continent_code;
debug('Value of the Geoip_Continent_Code header: %s', header);
const continent = header || 'default';
const region = serverSettings.awsHosting.lambda.regionByContinent[continent];
const arn = serverSettings.awsHosting.lambda.arnByRegion[region];
debug('The chosen AWS Lambda is: %s', arn);
return {region, arn};
}
// Retrive one run by id
app.get('/api/runs/:id', function(req, res) {
var runId = req.params.id;
@@ -296,4 +349,4 @@ var ApiController = function(app) {
}
};
module.exports = ApiController;
module.exports = ApiController;
lib/server/controllers/awsApiController.js
+310
View File
@@ -0,0 +1,310 @@
var debug = require('debug')('ylt:server');
var Q = require('q');
var AWS = require('aws-sdk');
var ylt = require('../../index');
var ScreenshotHandler = require('../../screenshotHandler');
var RunsQueue = require('../datastores/runsQueue');
var RunsDatastore = require('../datastores/runsDatastore');
var serverSettings = (process.env.IS_TEST) ? require('../../../test/fixtures/settings.json') : require('../../../server_config/settings.json');
var ResultsDatastore = (serverSettings.awsHosting) ? require('../datastores/awsResultsDatastore') : require('../datastores/resultsDatastore');
var ApiController = function(app) {
'use strict';
var queue = new RunsQueue();
var runsDatastore = new RunsDatastore();
var resultsDatastore = new ResultsDatastore();
// Increase AWS Lambda timeout
AWS.config.update({httpOptions: {timeout: 300000}});
// Create a new run
app.post('/api/runs', function(req, res) {
// Add http to the test URL
if (req.body.url && req.body.url.toLowerCase().indexOf('http://') !== 0 && req.body.url.toLowerCase().indexOf('https://') !== 0) {
req.body.url = 'http://' + req.body.url;
}
// Block requests to unwanted websites (=spam)
if (req.body.url && isBlocked(req.body.url)) {
console.error('Test blocked for URL: %s', req.body.url);
res.status(403).send('Forbidden');
return;
}
// Grab the test parameters and generate a random run ID
var run = {
runId: (Date.now()*1000 + Math.round(Math.random()*1000)).toString(36),
params: {
url: req.body.url,
waitForResponse: req.body.waitForResponse === true || req.body.waitForResponse === 'true' || req.body.waitForResponse === 1,
partialResult: req.body.partialResult || null,
screenshot: req.body.screenshot || false,
device: req.body.device || 'desktop',
proxy: req.body.proxy || null,
waitForSelector: req.body.waitForSelector || null,
cookie: req.body.cookie || null,
authUser: req.body.authUser || null,
authPass: req.body.authPass || null,
blockDomain: req.body.blockDomain || null,
allowDomain: req.body.allowDomain || null,
noExternals: req.body.noExternals || false
}
};
// Add test to the testQueue
debug('Adding test %s to the queue', run.runId);
var queuePromise = queue.push(run.runId);
// Save the run to the datastore
//runsDatastore.add(run, queuePromise.startingPosition);
runsDatastore.add(run, 0);
// Let's start the run
queuePromise.then(function() {
runsDatastore.updatePosition(run.runId, 0);
console.log('Launching test ' + run.runId + ' on ' + run.params.url);
var runOptions = {
screenshot: run.params.screenshot ? ScreenshotHandler.getTmpFileRelativePath() : false,
device: run.params.device,
proxy: run.params.proxy,
waitForSelector: run.params.waitForSelector,
cookie: run.params.cookie,
authUser: run.params.authUser,
authPass: run.params.authPass,
blockDomain: run.params.blockDomain,
allowDomain: run.params.allowDomain,
noExternals: run.params.noExternals
};
const {region, arn} = chooseLambdaRegionByGeoIP(req.headers);
const lambda = new AWS.Lambda({region: region});
return lambda.invoke({
FunctionName: arn,
InvocationType: 'RequestResponse',
Payload: JSON.stringify({url: run.params.url, id: run.runId, options: runOptions})
}).promise();
})
.then(function(response) {
debug('We\'ve got a response from AWS Lambda');
debug('StatusCode = %d', response.StatusCode);
debug('Payload = %s', response.Payload);
if (response.StatusCode === 200 && response.Payload && response.Payload !== 'null') {
const payload = JSON.parse(response.Payload);
if (payload.errorType) {
runsDatastore.markAsFailed(run.runId, payload.errorMessage);
} else {
debug('Success!');
runsDatastore.markAsComplete(run.runId);
}
} else {
debug('Empty response from the lambda agent');
runsDatastore.markAsFailed(run.runId, "Empty response from the agent");
}
})
.catch(function(err) {
debug('Error from AWS Lambda:');
debug(err);
runsDatastore.markAsFailed(run.runId, err.toString());
});
// The user doesn't want to wait for the response, sending the run ID only
debug('Sending response without waiting.');
res.setHeader('Content-Type', 'application/json');
res.send(JSON.stringify({runId: run.runId}));
});
// Reads the Geoip_Continent_Code header and chooses the right region from the settings
function chooseLambdaRegionByGeoIP(headers) {
// The settings can be configured like this in server_config/settings.json:
//
// "awsHosting": {
// "lambda": {
// "regionByContinent": {
// "AF": "eu-west-3",
// "AS": "ap-southeast-1",
// "EU": "eu-west-3",
// "NA": "us-east-1",
// "OC": "ap-southeast-1",
// "SA": "us-east-1",
// "default": "eu-west-3"
// },
// "arnByRegion": {
// "us-east-1": "arn:aws:lambda:us-east-1:xxx:function:xxx",
// "eu-west-3": "arn:aws:lambda:eu-west-3:xxx:function:xxx",
// "ap-southeast-1": "arn:aws:lambda:ap-southeast-1:xxx:function:xxx"
// }
// }
// },
const header = headers.geoip_continent_code;
debug('Value of the Geoip_Continent_Code header: %s', header);
const continent = header || 'default';
const region = serverSettings.awsHosting.lambda.regionByContinent[continent];
const arn = serverSettings.awsHosting.lambda.arnByRegion[region];
debug('The chosen AWS Lambda is: %s', arn);
return {region, arn};
}
// Retrive one run by id
app.get('/api/runs/:id', function(req, res) {
var runId = req.params.id;
var run = runsDatastore.get(runId);
if (run) {
res.setHeader('Content-Type', 'application/json');
res.send(JSON.stringify(run, null, 2));
} else {
res.status(404).send('Not found');
}
});
// Counts all pending runs
app.get('/api/runs', function(req, res) {
res.setHeader('Content-Type', 'application/json');
res.send(JSON.stringify({
pendingRuns: queue.length(),
timeSinceLastTestStarted: queue.timeSinceLastTestStarted()
}, null, 2));
});
// Delete one run by id
/*app.delete('/api/runs/:id', function(req, res) {
deleteRun()
});*/
// Delete all
/*app.delete('/api/runs', function(req, res) {
purgeRuns()
});
// List all
app.get('/api/runs', function(req, res) {
listRuns()
});
// Exists
app.head('/api/runs/:id', function(req, res) {
existsX();
// Returns 200 if the result exists or 404 if not
});
*/
// Retrive one result by id
app.get('/api/results/:id', function(req, res) {
getPartialResults(req.params.id, res, function(data) {
// Some fields can be excluded from the response, this way:
// /api/results/:id?exclude=field1,field2
if (req.query.exclude && typeof req.query.exclude === 'string') {
var excludedFields = req.query.exclude.split(',');
excludedFields.forEach(function(fieldName) {
if (data[fieldName]) {
delete data[fieldName];
}
});
}
return data;
});
});
// Retrieve one result and return only the generalScores part of the response
app.get('/api/results/:id/generalScores', function(req, res) {
getPartialResults(req.params.id, res, function(data) {
return data.scoreProfiles.generic;
});
});
app.get('/api/results/:id/generalScores/:scoreProfile', function(req, res) {
getPartialResults(req.params.id, res, function(data) {
return data.scoreProfiles[req.params.scoreProfile];
});
});
app.get('/api/results/:id/rules', function(req, res) {
getPartialResults(req.params.id, res, function(data) {
return data.rules;
});
});
app.get('/api/results/:id/javascriptExecutionTree', function(req, res) {
getPartialResults(req.params.id, res, function(data) {
return data.javascriptExecutionTree;
});
});
app.get('/api/results/:id/toolsResults/phantomas', function(req, res) {
getPartialResults(req.params.id, res, function(data) {
return data.toolsResults.phantomas;
});
});
function getPartialResults(runId, res, partialGetterFn) {
resultsDatastore.getResult(runId)
.then(function(data) {
var results = partialGetterFn(data);
if (typeof results === 'undefined') {
res.status(404).send('Not found');
return;
}
// TODO: Reverse quick fix
results.screenshotUrl = '/api/results/' + runId + '/screenshot.jpg';
res.setHeader('Content-Type', 'application/json');
res.send(JSON.stringify(results, null, 2));
}).fail(function() {
res.status(404).send('Not found');
});
}
// Retrive one result by id
app.get('/api/results/:id/screenshot.jpg', function(req, res) {
var runId = req.params.id;
resultsDatastore.getScreenshot(runId)
.then(function(screenshotBuffer) {
res.setHeader('Content-Type', 'image/jpeg');
res.send(screenshotBuffer);
}).fail(function() {
res.status(404).send('Not found');
});
});
function isBlocked(url) {
if (!serverSettings.blockedUrls) {
return false;
}
return serverSettings.blockedUrls.some(function(blockedUrl) {
return (url.indexOf(blockedUrl) === 0);
});
}
};
module.exports = ApiController;