Two version of the API: normal/lambda. Will be merged later
This commit is contained in:
Gaël Métais
@@ -1,16 +1,14 @@
|
||||
var debug = require('debug')('ylt:server');
|
||||
var Q = require('q');
|
||||
var AWS = require('aws-sdk');
|
||||
|
||||
var ylt = require('../../index');
|
||||
var ScreenshotHandler = require('../../screenshotHandler');
|
||||
var RunsQueue = require('../datastores/runsQueue');
|
||||
var RunsDatastore = require('../datastores/runsDatastore');
|
||||
var ResultsDatastore = require('../datastores/resultsDatastore');
|
||||
|
||||
var serverSettings = (process.env.IS_TEST) ? require('../../../test/fixtures/settings.json') : require('../../../server_config/settings.json');
|
||||
|
||||
var ResultsDatastore = (serverSettings.awsHosting) ? require('../datastores/awsResultsDatastore') : require('../datastores/resultsDatastore');
|
||||
|
||||
var ApiController = function(app) {
|
||||
'use strict';
|
||||
|
||||
@@ -53,13 +51,21 @@ var ApiController = function(app) {
|
||||
}
|
||||
};
|
||||
|
||||
// Create the tmp folder if it doesn't exist
|
||||
ScreenshotHandler.createTmpScreenshotFolder(run.runId);
|
||||
|
||||
// Add test to the testQueue
|
||||
debug('Adding test %s to the queue', run.runId);
|
||||
var queuePromise = queue.push(run.runId);
|
||||
|
||||
// Save the run to the datastore
|
||||
//runsDatastore.add(run, queuePromise.startingPosition);
|
||||
runsDatastore.add(run, 0);
|
||||
runsDatastore.add(run, queuePromise.startingPosition);
|
||||
|
||||
|
||||
// Listening for position updates
|
||||
queuePromise.progress(function(position) {
|
||||
runsDatastore.updatePosition(run.runId, position);
|
||||
});
|
||||
|
||||
// Let's start the run
|
||||
queuePromise.then(function() {
|
||||
@@ -81,82 +87,129 @@ var ApiController = function(app) {
|
||||
noExternals: run.params.noExternals
|
||||
};
|
||||
|
||||
const {region, arn} = chooseLambdaRegionByGeoIP(req.headers);
|
||||
const lambda = new AWS.Lambda({region: region});
|
||||
return ylt(run.params.url, runOptions)
|
||||
|
||||
// Update the progress bar on each progress
|
||||
.progress(function(progress) {
|
||||
runsDatastore.updateRunProgress(run.runId, progress);
|
||||
});
|
||||
|
||||
})
|
||||
|
||||
// Phantomas completed
|
||||
.then(function(data) {
|
||||
|
||||
debug('Success');
|
||||
data.runId = run.runId;
|
||||
|
||||
|
||||
return lambda.invoke({
|
||||
FunctionName: arn,
|
||||
InvocationType: 'RequestResponse',
|
||||
Payload: JSON.stringify({url: run.params.url, id: run.runId, options: runOptions})
|
||||
}).promise();
|
||||
// Some conditional steps exist if there is a screenshot
|
||||
var screenshotPromise = Q.resolve();
|
||||
|
||||
})
|
||||
if (run.params.screenshot) {
|
||||
|
||||
.then(function(response) {
|
||||
debug('We\'ve got a response from AWS Lambda');
|
||||
debug('StatusCode = %d', response.StatusCode);
|
||||
debug('Payload = %s', response.Payload);
|
||||
var screenshotSize = serverSettings.screenshotWidth ? serverSettings.screenshotWidth[run.params.device] : 400;
|
||||
|
||||
// Replace the empty promise created earlier with Q.resolve()
|
||||
screenshotPromise = ScreenshotHandler.findAndOptimizeScreenshot(screenshotSize)
|
||||
|
||||
// Read screenshot
|
||||
.then(function(screenshotBuffer) {
|
||||
if (screenshotBuffer) {
|
||||
debug('Image optimized');
|
||||
data.screenshotBuffer = screenshotBuffer;
|
||||
data.screenshotUrl = '/api/results/' + data.runId + '/screenshot.jpg';
|
||||
}
|
||||
})
|
||||
|
||||
// Don't worry if there's an error
|
||||
.fail(function(err) {
|
||||
debug('An error occured while creating the screenshot\'s thumbnail. Ignoring and continuing...');
|
||||
debug(err);
|
||||
});
|
||||
|
||||
if (response.StatusCode === 200 && response.Payload && response.Payload !== 'null') {
|
||||
debug('Success!');
|
||||
runsDatastore.markAsComplete(run.runId);
|
||||
} else {
|
||||
debug('Empty response from the lambda agent');
|
||||
runsDatastore.markAsFailed(run.runId, "Empty response from the agent");
|
||||
}
|
||||
|
||||
// Let's continue
|
||||
return screenshotPromise
|
||||
|
||||
// Save results
|
||||
.then(function() {
|
||||
// Remove uneeded temp screenshot path
|
||||
delete data.params.options.screenshot;
|
||||
|
||||
// Here we can remove tools results if not needed
|
||||
delete data.toolsResults.phantomas.offenders.requests;
|
||||
|
||||
return resultsDatastore.saveResult(data);
|
||||
})
|
||||
|
||||
// Mark as the run as complete and send the response if the request is still waiting
|
||||
.then(function() {
|
||||
|
||||
debug('Result saved in datastore');
|
||||
|
||||
runsDatastore.markAsComplete(run.runId);
|
||||
|
||||
if (run.params.waitForResponse) {
|
||||
|
||||
// If the user only wants a portion of the result (partialResult option)
|
||||
switch(run.params.partialResult) {
|
||||
case 'generalScores':
|
||||
res.redirect(302, '/api/results/' + run.runId + '/generalScores');
|
||||
break;
|
||||
case 'rules':
|
||||
res.redirect(302, '/api/results/' + run.runId + '/rules');
|
||||
break;
|
||||
case 'javascriptExecutionTree':
|
||||
res.redirect(302, '/api/results/' + run.runId + '/javascriptExecutionTree');
|
||||
break;
|
||||
case 'phantomas':
|
||||
res.redirect(302, '/api/results/' + run.runId + '/toolsResults/phantomas');
|
||||
break;
|
||||
default:
|
||||
res.redirect(302, '/api/results/' + run.runId);
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
.fail(function(err) {
|
||||
console.error('Test failed for URL: %s', run.params.url);
|
||||
console.error(err.toString());
|
||||
|
||||
runsDatastore.markAsFailed(run.runId, err.toString());
|
||||
|
||||
res.status(500).send('An error occured');
|
||||
});
|
||||
|
||||
})
|
||||
|
||||
.catch(function(err) {
|
||||
debug('Error from AWS Lambda:');
|
||||
debug(err);
|
||||
.fail(function(err) {
|
||||
|
||||
console.error('Test failed for URL: %s', run.params.url);
|
||||
console.error(err.toString());
|
||||
|
||||
runsDatastore.markAsFailed(run.runId, err.toString());
|
||||
|
||||
res.status(400).send('Bad request');
|
||||
|
||||
})
|
||||
|
||||
.finally(function() {
|
||||
queue.remove(run.runId);
|
||||
});
|
||||
|
||||
|
||||
// The user doesn't want to wait for the response, sending the run ID only
|
||||
debug('Sending response without waiting.');
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
res.send(JSON.stringify({runId: run.runId}));
|
||||
if (!run.params.waitForResponse) {
|
||||
debug('Sending response without waiting.');
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
res.send(JSON.stringify({runId: run.runId}));
|
||||
}
|
||||
|
||||
});
|
||||
|
||||
|
||||
// Reads the Geoip_Continent_Code header and chooses the right region from the settings
|
||||
function chooseLambdaRegionByGeoIP(headers) {
|
||||
|
||||
// The settings can be configured like this in server_config/settings.json:
|
||||
//
|
||||
// "awsHosting": {
|
||||
// "lambda": {
|
||||
// "regionByContinent": {
|
||||
// "AF": "eu-west-3",
|
||||
// "AS": "ap-southeast-1",
|
||||
// "EU": "eu-west-3",
|
||||
// "NA": "us-east-1",
|
||||
// "OC": "ap-southeast-1",
|
||||
// "SA": "us-east-1",
|
||||
// "default": "eu-west-3"
|
||||
// },
|
||||
// "arnByRegion": {
|
||||
// "us-east-1": "arn:aws:lambda:us-east-1:xxx:function:xxx",
|
||||
// "eu-west-3": "arn:aws:lambda:eu-west-3:xxx:function:xxx",
|
||||
// "ap-southeast-1": "arn:aws:lambda:ap-southeast-1:xxx:function:xxx"
|
||||
// }
|
||||
// }
|
||||
// },
|
||||
|
||||
const header = headers.geoip_continent_code;
|
||||
debug('Value of the Geoip_Continent_Code header: %s', header);
|
||||
|
||||
const continent = header || 'default';
|
||||
const region = serverSettings.awsHosting.lambda.regionByContinent[continent];
|
||||
const arn = serverSettings.awsHosting.lambda.arnByRegion[region];
|
||||
debug('The chosen AWS Lambda is: %s', arn);
|
||||
|
||||
return {region, arn};
|
||||
}
|
||||
|
||||
|
||||
// Retrive one run by id
|
||||
app.get('/api/runs/:id', function(req, res) {
|
||||
var runId = req.params.id;
|
||||
@@ -296,4 +349,4 @@ var ApiController = function(app) {
|
||||
}
|
||||
};
|
||||
|
||||
module.exports = ApiController;
|
||||
module.exports = ApiController;
|
||||
Reference in New Issue
Block a user