Merge split into aws branch
This commit is contained in:
Gaël Métais
+4
-1
@@ -4,6 +4,8 @@ var Q = require('q');
|
||||
var Runner = require('./runner');
|
||||
var ScreenshotHandler = require('./screenshotHandler');
|
||||
|
||||
var packageJson = require('../package.json');
|
||||
|
||||
|
||||
var yellowLabTools = function(url, options) {
|
||||
var deferred = Q.defer();
|
||||
@@ -76,4 +78,5 @@ var yellowLabTools = function(url, options) {
|
||||
return deferred.promise;
|
||||
};
|
||||
|
||||
module.exports = yellowLabTools;
|
||||
module.exports = yellowLabTools;
|
||||
module.exports.version = packageJson.version;
|
||||
@@ -1,123 +0,0 @@
|
||||
var debug = require('debug')('ylt:screenshotHandler');
|
||||
var Jimp = require('jimp');
|
||||
var Q = require('q');
|
||||
var fs = require('fs');
|
||||
var path = require('path');
|
||||
|
||||
var serverSettings = require('../server_config/settings.json');
|
||||
|
||||
|
||||
var screenshotHandler = function() {
|
||||
|
||||
|
||||
this.findAndOptimizeScreenshot = function(width) {
|
||||
var that = this;
|
||||
|
||||
debug('Starting screenshot transformation');
|
||||
|
||||
return this.openImage(this.getTmpFileRelativePath())
|
||||
|
||||
.then(function(image) {
|
||||
that.deleteTmpFile(that.getTmpFileRelativePath());
|
||||
return that.resizeImage(image, width);
|
||||
})
|
||||
|
||||
.then(this.toBuffer);
|
||||
};
|
||||
|
||||
|
||||
this.openImage = function(imagePath) {
|
||||
var deferred = Q.defer();
|
||||
|
||||
Jimp.read(imagePath, function(err, image){
|
||||
if (err) {
|
||||
debug('Could not open imagePath %s', imagePath);
|
||||
debug(err);
|
||||
|
||||
deferred.reject(err);
|
||||
} else {
|
||||
debug('Image correctly open');
|
||||
deferred.resolve(image);
|
||||
}
|
||||
});
|
||||
|
||||
return deferred.promise;
|
||||
};
|
||||
|
||||
|
||||
this.resizeImage = function(image, newWidth) {
|
||||
var deferred = Q.defer();
|
||||
|
||||
var currentWidth = image.bitmap.width;
|
||||
|
||||
if (currentWidth > 0) {
|
||||
var ratio = newWidth / currentWidth;
|
||||
|
||||
image.scale(ratio, function(err, image){
|
||||
if (err) {
|
||||
debug('Could not resize image');
|
||||
debug(err);
|
||||
|
||||
deferred.reject(err);
|
||||
} else {
|
||||
debug('Image correctly resized');
|
||||
deferred.resolve(image);
|
||||
}
|
||||
});
|
||||
} else {
|
||||
deferred.reject('Could not resize an empty image');
|
||||
}
|
||||
|
||||
return deferred.promise;
|
||||
};
|
||||
|
||||
|
||||
this.toBuffer = function(image) {
|
||||
var deferred = Q.defer();
|
||||
|
||||
image.quality(85).getBuffer(Jimp.MIME_JPEG, function(err, buffer){
|
||||
if (err) {
|
||||
debug('Could not save image to buffer');
|
||||
debug(err);
|
||||
|
||||
deferred.reject(err);
|
||||
} else {
|
||||
debug('Image correctly transformed to buffer');
|
||||
deferred.resolve(buffer);
|
||||
}
|
||||
});
|
||||
|
||||
return deferred.promise;
|
||||
};
|
||||
|
||||
|
||||
this.deleteTmpFile = function(tmpFilePath) {
|
||||
var deferred = Q.defer();
|
||||
|
||||
//fs.unlink(this.getTmpFileRelativePath(), function (err) {
|
||||
// if (err) {
|
||||
// debug('Screenshot temporary file not found, could not be deleted. But it is not a problem.');
|
||||
// } else {
|
||||
// debug('Screenshot temporary file deleted.');
|
||||
// }
|
||||
|
||||
deferred.resolve();
|
||||
//});
|
||||
|
||||
return deferred.promise;
|
||||
};
|
||||
|
||||
|
||||
this.getTmpFileRelativePath = function() {
|
||||
|
||||
// Chrome saves a temporary file on the disk, which is then removed.
|
||||
// Its default folder is /tmp, but it can be changed in server_config/settings.json
|
||||
var tmpFolderPath = serverSettings.screenshotTempPath || '/tmp';
|
||||
var tmpFileName = 'temp-chrome-screenshot.png';
|
||||
var tmpFileFullPath = path.join(tmpFolderPath, tmpFileName);
|
||||
|
||||
return tmpFileFullPath;
|
||||
};
|
||||
};
|
||||
|
||||
module.exports = new screenshotHandler();
|
||||
@@ -1,352 +0,0 @@
|
||||
var debug = require('debug')('ylt:server');
|
||||
var Q = require('q');
|
||||
|
||||
var ylt = require('../../index');
|
||||
var ScreenshotHandler = require('../../screenshotHandler');
|
||||
var RunsQueue = require('../datastores/runsQueue');
|
||||
var RunsDatastore = require('../datastores/runsDatastore');
|
||||
var ResultsDatastore = require('../datastores/resultsDatastore');
|
||||
|
||||
var serverSettings = (process.env.IS_TEST) ? require('../../../test/fixtures/settings.json') : require('../../../server_config/settings.json');
|
||||
|
||||
var ApiController = function(app) {
|
||||
'use strict';
|
||||
|
||||
var queue = new RunsQueue();
|
||||
var runsDatastore = new RunsDatastore();
|
||||
var resultsDatastore = new ResultsDatastore();
|
||||
|
||||
// Create a new run
|
||||
app.post('/api/runs', function(req, res) {
|
||||
|
||||
// Add https to the test URL
|
||||
if (req.body.url && req.body.url.toLowerCase().indexOf('http://') !== 0 && req.body.url.toLowerCase().indexOf('https://') !== 0) {
|
||||
req.body.url = 'https://' + req.body.url;
|
||||
}
|
||||
|
||||
// Block requests to unwanted websites (=spam)
|
||||
if (req.body.url && isBlocked(req.body.url)) {
|
||||
console.error('Test blocked for URL: %s', req.body.url);
|
||||
res.status(403).send('Forbidden');
|
||||
return;
|
||||
}
|
||||
|
||||
// Grab the test parameters and generate a random run ID
|
||||
var run = {
|
||||
runId: (Date.now()*1000 + Math.round(Math.random()*1000)).toString(36),
|
||||
params: {
|
||||
url: req.body.url,
|
||||
waitForResponse: req.body.waitForResponse === true || req.body.waitForResponse === 'true' || req.body.waitForResponse === 1,
|
||||
partialResult: req.body.partialResult || null,
|
||||
screenshot: req.body.screenshot || false,
|
||||
device: req.body.device || 'desktop',
|
||||
proxy: req.body.proxy || null,
|
||||
waitForSelector: req.body.waitForSelector || null,
|
||||
cookie: req.body.cookie || null,
|
||||
authUser: req.body.authUser || null,
|
||||
authPass: req.body.authPass || null,
|
||||
blockDomain: req.body.blockDomain || null,
|
||||
allowDomain: req.body.allowDomain || null,
|
||||
noExternals: req.body.noExternals || false
|
||||
}
|
||||
};
|
||||
|
||||
// Create the tmp folder if it doesn't exist
|
||||
ScreenshotHandler.createTmpScreenshotFolder(run.runId);
|
||||
|
||||
// Add test to the testQueue
|
||||
debug('Adding test %s to the queue', run.runId);
|
||||
var queuePromise = queue.push(run.runId);
|
||||
|
||||
// Save the run to the datastore
|
||||
runsDatastore.add(run, queuePromise.startingPosition);
|
||||
|
||||
|
||||
// Listening for position updates
|
||||
queuePromise.progress(function(position) {
|
||||
runsDatastore.updatePosition(run.runId, position);
|
||||
});
|
||||
|
||||
// Let's start the run
|
||||
queuePromise.then(function() {
|
||||
|
||||
runsDatastore.updatePosition(run.runId, 0);
|
||||
|
||||
console.log('Launching test ' + run.runId + ' on ' + run.params.url);
|
||||
|
||||
var runOptions = {
|
||||
screenshot: run.params.screenshot ? ScreenshotHandler.getTmpFileRelativePath() : false,
|
||||
device: run.params.device,
|
||||
proxy: run.params.proxy,
|
||||
waitForSelector: run.params.waitForSelector,
|
||||
cookie: run.params.cookie,
|
||||
authUser: run.params.authUser,
|
||||
authPass: run.params.authPass,
|
||||
blockDomain: run.params.blockDomain,
|
||||
allowDomain: run.params.allowDomain,
|
||||
noExternals: run.params.noExternals
|
||||
};
|
||||
|
||||
return ylt(run.params.url, runOptions)
|
||||
|
||||
// Update the progress bar on each progress
|
||||
.progress(function(progress) {
|
||||
runsDatastore.updateRunProgress(run.runId, progress);
|
||||
});
|
||||
|
||||
})
|
||||
|
||||
// Phantomas completed
|
||||
.then(function(data) {
|
||||
|
||||
debug('Success');
|
||||
data.runId = run.runId;
|
||||
|
||||
|
||||
// Some conditional steps exist if there is a screenshot
|
||||
var screenshotPromise = Q.resolve();
|
||||
|
||||
if (run.params.screenshot) {
|
||||
|
||||
var screenshotSize = serverSettings.screenshotWidth ? serverSettings.screenshotWidth[run.params.device] : 400;
|
||||
|
||||
// Replace the empty promise created earlier with Q.resolve()
|
||||
screenshotPromise = ScreenshotHandler.findAndOptimizeScreenshot(screenshotSize)
|
||||
|
||||
// Read screenshot
|
||||
.then(function(screenshotBuffer) {
|
||||
if (screenshotBuffer) {
|
||||
debug('Image optimized');
|
||||
data.screenshotBuffer = screenshotBuffer;
|
||||
data.screenshotUrl = '/api/results/' + data.runId + '/screenshot.jpg';
|
||||
}
|
||||
})
|
||||
|
||||
// Don't worry if there's an error
|
||||
.fail(function(err) {
|
||||
debug('An error occured while creating the screenshot\'s thumbnail. Ignoring and continuing...');
|
||||
debug(err);
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
// Let's continue
|
||||
return screenshotPromise
|
||||
|
||||
// Save results
|
||||
.then(function() {
|
||||
// Remove uneeded temp screenshot path
|
||||
delete data.params.options.screenshot;
|
||||
|
||||
// Here we can remove tools results if not needed
|
||||
delete data.toolsResults.phantomas.offenders.requests;
|
||||
|
||||
return resultsDatastore.saveResult(data);
|
||||
})
|
||||
|
||||
// Mark as the run as complete and send the response if the request is still waiting
|
||||
.then(function() {
|
||||
|
||||
debug('Result saved in datastore');
|
||||
|
||||
runsDatastore.markAsComplete(run.runId);
|
||||
|
||||
if (run.params.waitForResponse) {
|
||||
|
||||
// If the user only wants a portion of the result (partialResult option)
|
||||
switch(run.params.partialResult) {
|
||||
case 'generalScores':
|
||||
res.redirect(302, '/api/results/' + run.runId + '/generalScores');
|
||||
break;
|
||||
case 'rules':
|
||||
res.redirect(302, '/api/results/' + run.runId + '/rules');
|
||||
break;
|
||||
case 'javascriptExecutionTree':
|
||||
res.redirect(302, '/api/results/' + run.runId + '/javascriptExecutionTree');
|
||||
break;
|
||||
case 'phantomas':
|
||||
res.redirect(302, '/api/results/' + run.runId + '/toolsResults/phantomas');
|
||||
break;
|
||||
default:
|
||||
res.redirect(302, '/api/results/' + run.runId);
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
.fail(function(err) {
|
||||
console.error('Test failed for URL: %s', run.params.url);
|
||||
console.error(err.toString());
|
||||
|
||||
runsDatastore.markAsFailed(run.runId, err.toString());
|
||||
|
||||
res.status(500).send('An error occured');
|
||||
});
|
||||
|
||||
})
|
||||
|
||||
.fail(function(err) {
|
||||
|
||||
console.error('Test failed for URL: %s', run.params.url);
|
||||
console.error(err.toString());
|
||||
|
||||
runsDatastore.markAsFailed(run.runId, err.toString());
|
||||
|
||||
res.status(400).send('Bad request');
|
||||
|
||||
})
|
||||
|
||||
.finally(function() {
|
||||
queue.remove(run.runId);
|
||||
});
|
||||
|
||||
|
||||
// The user doesn't want to wait for the response, sending the run ID only
|
||||
if (!run.params.waitForResponse) {
|
||||
debug('Sending response without waiting.');
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
res.send(JSON.stringify({runId: run.runId}));
|
||||
}
|
||||
|
||||
});
|
||||
|
||||
|
||||
// Retrive one run by id
|
||||
app.get('/api/runs/:id', function(req, res) {
|
||||
var runId = req.params.id;
|
||||
|
||||
var run = runsDatastore.get(runId);
|
||||
|
||||
if (run) {
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
res.send(JSON.stringify(run, null, 2));
|
||||
} else {
|
||||
res.status(404).send('Not found');
|
||||
}
|
||||
});
|
||||
|
||||
// Counts all pending runs
|
||||
app.get('/api/runs', function(req, res) {
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
res.send(JSON.stringify({
|
||||
pendingRuns: queue.length(),
|
||||
timeSinceLastTestStarted: queue.timeSinceLastTestStarted()
|
||||
}, null, 2));
|
||||
});
|
||||
|
||||
// Delete one run by id
|
||||
/*app.delete('/api/runs/:id', function(req, res) {
|
||||
deleteRun()
|
||||
});*/
|
||||
|
||||
// Delete all
|
||||
/*app.delete('/api/runs', function(req, res) {
|
||||
purgeRuns()
|
||||
});
|
||||
|
||||
// List all
|
||||
app.get('/api/runs', function(req, res) {
|
||||
listRuns()
|
||||
});
|
||||
|
||||
// Exists
|
||||
app.head('/api/runs/:id', function(req, res) {
|
||||
existsX();
|
||||
// Returns 200 if the result exists or 404 if not
|
||||
});
|
||||
*/
|
||||
|
||||
// Retrive one result by id
|
||||
app.get('/api/results/:id', function(req, res) {
|
||||
getPartialResults(req.params.id, res, function(data) {
|
||||
|
||||
// Some fields can be excluded from the response, this way:
|
||||
// /api/results/:id?exclude=field1,field2
|
||||
if (req.query.exclude && typeof req.query.exclude === 'string') {
|
||||
var excludedFields = req.query.exclude.split(',');
|
||||
excludedFields.forEach(function(fieldName) {
|
||||
if (data[fieldName]) {
|
||||
delete data[fieldName];
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
return data;
|
||||
});
|
||||
});
|
||||
|
||||
// Retrieve one result and return only the generalScores part of the response
|
||||
app.get('/api/results/:id/generalScores', function(req, res) {
|
||||
getPartialResults(req.params.id, res, function(data) {
|
||||
return data.scoreProfiles.generic;
|
||||
});
|
||||
});
|
||||
|
||||
app.get('/api/results/:id/generalScores/:scoreProfile', function(req, res) {
|
||||
getPartialResults(req.params.id, res, function(data) {
|
||||
return data.scoreProfiles[req.params.scoreProfile];
|
||||
});
|
||||
});
|
||||
|
||||
app.get('/api/results/:id/rules', function(req, res) {
|
||||
getPartialResults(req.params.id, res, function(data) {
|
||||
return data.rules;
|
||||
});
|
||||
});
|
||||
|
||||
app.get('/api/results/:id/javascriptExecutionTree', function(req, res) {
|
||||
getPartialResults(req.params.id, res, function(data) {
|
||||
return data.javascriptExecutionTree;
|
||||
});
|
||||
});
|
||||
|
||||
app.get('/api/results/:id/toolsResults/phantomas', function(req, res) {
|
||||
getPartialResults(req.params.id, res, function(data) {
|
||||
return data.toolsResults.phantomas;
|
||||
});
|
||||
});
|
||||
|
||||
function getPartialResults(runId, res, partialGetterFn) {
|
||||
resultsDatastore.getResult(runId)
|
||||
.then(function(data) {
|
||||
var results = partialGetterFn(data);
|
||||
|
||||
if (typeof results === 'undefined') {
|
||||
res.status(404).send('Not found');
|
||||
return;
|
||||
}
|
||||
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
res.send(JSON.stringify(results, null, 2));
|
||||
|
||||
}).fail(function() {
|
||||
res.status(404).send('Not found');
|
||||
});
|
||||
}
|
||||
|
||||
// Retrive one result by id
|
||||
app.get('/api/results/:id/screenshot.jpg', function(req, res) {
|
||||
var runId = req.params.id;
|
||||
|
||||
resultsDatastore.getScreenshot(runId)
|
||||
.then(function(screenshotBuffer) {
|
||||
|
||||
res.setHeader('Content-Type', 'image/jpeg');
|
||||
res.send(screenshotBuffer);
|
||||
|
||||
}).fail(function() {
|
||||
res.status(404).send('Not found');
|
||||
});
|
||||
});
|
||||
|
||||
function isBlocked(url) {
|
||||
if (!serverSettings.blockedUrls) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return serverSettings.blockedUrls.some(function(blockedUrl) {
|
||||
return (url.indexOf(blockedUrl) === 0);
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
module.exports = ApiController;
|
||||
@@ -1,312 +0,0 @@
|
||||
var debug = require('debug')('ylt:server');
|
||||
var Q = require('q');
|
||||
var AWS = require('aws-sdk');
|
||||
|
||||
var ylt = require('../../index');
|
||||
var ScreenshotHandler = require('../../screenshotHandler');
|
||||
var RunsQueue = require('../datastores/runsQueue');
|
||||
var RunsDatastore = require('../datastores/runsDatastore');
|
||||
|
||||
var serverSettings = (process.env.IS_TEST) ? require('../../../test/fixtures/settings.json') : require('../../../server_config/settings.json');
|
||||
|
||||
var ResultsDatastore = (serverSettings.awsHosting) ? require('../datastores/awsResultsDatastore') : require('../datastores/resultsDatastore');
|
||||
|
||||
var ApiController = function(app) {
|
||||
'use strict';
|
||||
|
||||
var queue = new RunsQueue();
|
||||
var runsDatastore = new RunsDatastore();
|
||||
var resultsDatastore = new ResultsDatastore();
|
||||
|
||||
// Increase AWS Lambda timeout
|
||||
AWS.config.update({httpOptions: {timeout: 300000}});
|
||||
|
||||
// Create a new run
|
||||
app.post('/api/runs', function(req, res) {
|
||||
|
||||
// Add http to the test URL
|
||||
if (req.body.url && req.body.url.toLowerCase().indexOf('http://') !== 0 && req.body.url.toLowerCase().indexOf('https://') !== 0) {
|
||||
req.body.url = 'https://' + req.body.url;
|
||||
}
|
||||
|
||||
// Block requests to unwanted websites (=spam)
|
||||
if (req.body.url && isBlocked(req.body.url)) {
|
||||
console.error('Test blocked for URL: %s', req.body.url);
|
||||
res.status(403).send('Forbidden');
|
||||
return;
|
||||
}
|
||||
|
||||
// Grab the test parameters and generate a random run ID
|
||||
var run = {
|
||||
runId: (Date.now()*1000 + Math.round(Math.random()*1000)).toString(36),
|
||||
params: {
|
||||
url: req.body.url,
|
||||
waitForResponse: req.body.waitForResponse === true || req.body.waitForResponse === 'true' || req.body.waitForResponse === 1,
|
||||
partialResult: req.body.partialResult || null,
|
||||
screenshot: req.body.screenshot || false,
|
||||
device: req.body.device || 'desktop',
|
||||
proxy: req.body.proxy || null,
|
||||
waitForSelector: req.body.waitForSelector || null,
|
||||
cookie: req.body.cookie || null,
|
||||
authUser: req.body.authUser || null,
|
||||
authPass: req.body.authPass || null,
|
||||
blockDomain: req.body.blockDomain || null,
|
||||
allowDomain: req.body.allowDomain || null,
|
||||
noExternals: req.body.noExternals || false
|
||||
}
|
||||
};
|
||||
|
||||
// Add test to the testQueue
|
||||
debug('Adding test %s to the queue', run.runId);
|
||||
var queuePromise = queue.push(run.runId);
|
||||
|
||||
// Save the run to the datastore
|
||||
//runsDatastore.add(run, queuePromise.startingPosition);
|
||||
runsDatastore.add(run, 0);
|
||||
|
||||
// Let's start the run
|
||||
queuePromise.then(function() {
|
||||
|
||||
runsDatastore.updatePosition(run.runId, 0);
|
||||
|
||||
console.log('Launching test ' + run.runId + ' on ' + run.params.url);
|
||||
|
||||
var runOptions = {
|
||||
screenshot: run.params.screenshot ? ScreenshotHandler.getTmpFileRelativePath() : false,
|
||||
device: run.params.device,
|
||||
proxy: run.params.proxy,
|
||||
waitForSelector: run.params.waitForSelector,
|
||||
cookie: run.params.cookie,
|
||||
authUser: run.params.authUser,
|
||||
authPass: run.params.authPass,
|
||||
blockDomain: run.params.blockDomain,
|
||||
allowDomain: run.params.allowDomain,
|
||||
noExternals: run.params.noExternals
|
||||
};
|
||||
|
||||
const {region, arn} = chooseLambdaRegionByGeoIP(req.headers);
|
||||
const lambda = new AWS.Lambda({region: region});
|
||||
|
||||
return lambda.invoke({
|
||||
FunctionName: arn,
|
||||
InvocationType: 'RequestResponse',
|
||||
Payload: JSON.stringify({url: run.params.url, id: run.runId, options: runOptions})
|
||||
}).promise();
|
||||
|
||||
})
|
||||
|
||||
.then(function(response) {
|
||||
debug('We\'ve got a response from AWS Lambda');
|
||||
debug('StatusCode = %d', response.StatusCode);
|
||||
debug('Payload = %s', response.Payload);
|
||||
|
||||
if (response.StatusCode === 200 && response.Payload && response.Payload !== 'null') {
|
||||
const payload = JSON.parse(response.Payload);
|
||||
if (payload.status === 'failed') {
|
||||
debug('Failed with error %s', payload.errorMessage);
|
||||
runsDatastore.markAsFailed(run.runId, payload.errorMessage);
|
||||
} else {
|
||||
debug('Success!');
|
||||
runsDatastore.markAsComplete(run.runId);
|
||||
}
|
||||
} else {
|
||||
debug('Empty response from the lambda agent');
|
||||
runsDatastore.markAsFailed(run.runId, "Empty response from the agent");
|
||||
}
|
||||
})
|
||||
|
||||
.catch(function(err) {
|
||||
debug('Error from AWS Lambda:');
|
||||
debug(err);
|
||||
|
||||
runsDatastore.markAsFailed(run.runId, err.toString());
|
||||
});
|
||||
|
||||
// The user doesn't want to wait for the response, sending the run ID only
|
||||
debug('Sending response without waiting.');
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
res.send(JSON.stringify({runId: run.runId}));
|
||||
|
||||
});
|
||||
|
||||
|
||||
// Reads the Geoip_Continent_Code header and chooses the right region from the settings
|
||||
function chooseLambdaRegionByGeoIP(headers) {
|
||||
|
||||
// The settings can be configured like this in server_config/settings.json:
|
||||
//
|
||||
// "awsHosting": {
|
||||
// "lambda": {
|
||||
// "regionByContinent": {
|
||||
// "AF": "eu-west-3",
|
||||
// "AS": "ap-southeast-1",
|
||||
// "EU": "eu-west-3",
|
||||
// "NA": "us-east-1",
|
||||
// "OC": "ap-southeast-1",
|
||||
// "SA": "us-east-1",
|
||||
// "default": "eu-west-3"
|
||||
// },
|
||||
// "arnByRegion": {
|
||||
// "us-east-1": "arn:aws:lambda:us-east-1:xxx:function:xxx",
|
||||
// "eu-west-3": "arn:aws:lambda:eu-west-3:xxx:function:xxx",
|
||||
// "ap-southeast-1": "arn:aws:lambda:ap-southeast-1:xxx:function:xxx"
|
||||
// }
|
||||
// }
|
||||
// },
|
||||
|
||||
const header = headers.geoip_continent_code;
|
||||
debug('Value of the Geoip_Continent_Code header: %s', header);
|
||||
|
||||
const continent = header || 'default';
|
||||
const region = serverSettings.awsHosting.lambda.regionByContinent[continent];
|
||||
const arn = serverSettings.awsHosting.lambda.arnByRegion[region];
|
||||
debug('The chosen AWS Lambda is: %s', arn);
|
||||
|
||||
return {region, arn};
|
||||
}
|
||||
|
||||
|
||||
// Retrive one run by id
|
||||
app.get('/api/runs/:id', function(req, res) {
|
||||
var runId = req.params.id;
|
||||
|
||||
var run = runsDatastore.get(runId);
|
||||
|
||||
if (run) {
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
res.send(JSON.stringify(run, null, 2));
|
||||
} else {
|
||||
res.status(404).send('Not found');
|
||||
}
|
||||
});
|
||||
|
||||
// Counts all pending runs
|
||||
app.get('/api/runs', function(req, res) {
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
res.send(JSON.stringify({
|
||||
pendingRuns: queue.length(),
|
||||
timeSinceLastTestStarted: queue.timeSinceLastTestStarted()
|
||||
}, null, 2));
|
||||
});
|
||||
|
||||
// Delete one run by id
|
||||
/*app.delete('/api/runs/:id', function(req, res) {
|
||||
deleteRun()
|
||||
});*/
|
||||
|
||||
// Delete all
|
||||
/*app.delete('/api/runs', function(req, res) {
|
||||
purgeRuns()
|
||||
});
|
||||
|
||||
// List all
|
||||
app.get('/api/runs', function(req, res) {
|
||||
listRuns()
|
||||
});
|
||||
|
||||
// Exists
|
||||
app.head('/api/runs/:id', function(req, res) {
|
||||
existsX();
|
||||
// Returns 200 if the result exists or 404 if not
|
||||
});
|
||||
*/
|
||||
|
||||
// Retrive one result by id
|
||||
app.get('/api/results/:id', function(req, res) {
|
||||
getPartialResults(req.params.id, res, function(data) {
|
||||
|
||||
// Some fields can be excluded from the response, this way:
|
||||
// /api/results/:id?exclude=field1,field2
|
||||
if (req.query.exclude && typeof req.query.exclude === 'string') {
|
||||
var excludedFields = req.query.exclude.split(',');
|
||||
excludedFields.forEach(function(fieldName) {
|
||||
if (data[fieldName]) {
|
||||
delete data[fieldName];
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
return data;
|
||||
});
|
||||
});
|
||||
|
||||
// Retrieve one result and return only the generalScores part of the response
|
||||
app.get('/api/results/:id/generalScores', function(req, res) {
|
||||
getPartialResults(req.params.id, res, function(data) {
|
||||
return data.scoreProfiles.generic;
|
||||
});
|
||||
});
|
||||
|
||||
app.get('/api/results/:id/generalScores/:scoreProfile', function(req, res) {
|
||||
getPartialResults(req.params.id, res, function(data) {
|
||||
return data.scoreProfiles[req.params.scoreProfile];
|
||||
});
|
||||
});
|
||||
|
||||
app.get('/api/results/:id/rules', function(req, res) {
|
||||
getPartialResults(req.params.id, res, function(data) {
|
||||
return data.rules;
|
||||
});
|
||||
});
|
||||
|
||||
app.get('/api/results/:id/javascriptExecutionTree', function(req, res) {
|
||||
getPartialResults(req.params.id, res, function(data) {
|
||||
return data.javascriptExecutionTree;
|
||||
});
|
||||
});
|
||||
|
||||
app.get('/api/results/:id/toolsResults/phantomas', function(req, res) {
|
||||
getPartialResults(req.params.id, res, function(data) {
|
||||
return data.toolsResults.phantomas;
|
||||
});
|
||||
});
|
||||
|
||||
function getPartialResults(runId, res, partialGetterFn) {
|
||||
resultsDatastore.getResult(runId)
|
||||
.then(function(data) {
|
||||
var results = partialGetterFn(data);
|
||||
|
||||
if (typeof results === 'undefined') {
|
||||
res.status(404).send('Not found');
|
||||
return;
|
||||
}
|
||||
|
||||
// Quickfix (TODO remove)
|
||||
results.runId = runId;
|
||||
results.screenshotUrl = '/api/results/' + runId + '/screenshot.jpg';
|
||||
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
res.send(JSON.stringify(results, null, 2));
|
||||
|
||||
}).fail(function() {
|
||||
res.status(404).send('Not found');
|
||||
});
|
||||
}
|
||||
|
||||
// Retrive one result by id
|
||||
app.get('/api/results/:id/screenshot.jpg', function(req, res) {
|
||||
var runId = req.params.id;
|
||||
|
||||
resultsDatastore.getScreenshot(runId)
|
||||
.then(function(screenshotBuffer) {
|
||||
|
||||
res.setHeader('Content-Type', 'image/jpeg');
|
||||
res.send(screenshotBuffer);
|
||||
|
||||
}).fail(function() {
|
||||
res.status(404).send('Not found');
|
||||
});
|
||||
});
|
||||
|
||||
function isBlocked(url) {
|
||||
if (!serverSettings.blockedUrls) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return serverSettings.blockedUrls.some(function(blockedUrl) {
|
||||
return (url.indexOf(blockedUrl) === 0);
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
module.exports = ApiController;
|
||||
@@ -1,45 +0,0 @@
|
||||
var path = require('path');
|
||||
var express = require('express');
|
||||
|
||||
var serverSettings = (process.env.IS_TEST) ? require('../../../test/fixtures/settings.json') : require('../../../server_config/settings.json');
|
||||
var packageJson = require('../../../package.json');
|
||||
|
||||
var FrontController = function(app) {
|
||||
'use strict';
|
||||
|
||||
var cacheDuration = 365 * 24 * 60 * 60 * 1000; // One year
|
||||
var assetsPath = (app.get('env') === 'development') ? '../../../front/src' : '../../../front/build';
|
||||
|
||||
// Routes templating
|
||||
var routes = ['/', '/about', '/result/:runId', '/result/:runId/screenshot', '/result/:runId/rule/:policy', '/queue/:runId'];
|
||||
|
||||
routes.forEach(function(route) {
|
||||
app.get(route, function(req, res) {
|
||||
res.setHeader('Cache-Control', 'public, max-age=20');
|
||||
res.render(path.join(__dirname, assetsPath, 'main.html'), {
|
||||
version: 'v' + packageJson.version,
|
||||
baseUrl: app.locals.baseUrl || '/',
|
||||
googleAnalyticsId: serverSettings.googleAnalyticsId,
|
||||
sponsoring: serverSettings.sponsoring || {}
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// Views templating
|
||||
app.get('/views/:viewName', function(req, res) {
|
||||
res.setHeader('Cache-Control', 'public, max-age=' + cacheDuration);
|
||||
res.render(path.join(__dirname, assetsPath, 'views/' + req.params.viewName), {
|
||||
baseUrl: app.locals.baseUrl || '/',
|
||||
sponsoring: serverSettings.sponsoring || {}
|
||||
});
|
||||
});
|
||||
|
||||
// Static assets
|
||||
app.use('/css', express.static(path.join(__dirname, assetsPath, 'css'), { maxAge: cacheDuration }));
|
||||
app.use('/fonts', express.static(path.join(__dirname, assetsPath, 'fonts'), { maxAge: cacheDuration }));
|
||||
app.use('/img', express.static(path.join(__dirname, assetsPath, 'img'), { maxAge: cacheDuration }));
|
||||
app.use('/js', express.static(path.join(__dirname, assetsPath, 'js'), { maxAge: cacheDuration }));
|
||||
app.use('/node_modules', express.static(path.join(__dirname, '../../../node_modules'), { maxAge: cacheDuration }));
|
||||
};
|
||||
|
||||
module.exports = FrontController;
|
||||
@@ -1,119 +0,0 @@
|
||||
const Q = require('q');
|
||||
const debug = require('debug')('ylt:resultsDatastore');
|
||||
const path = require('path');
|
||||
const AWS = require('aws-sdk');
|
||||
|
||||
|
||||
function ResultsDatastore() {
|
||||
'use strict';
|
||||
|
||||
const serverSettings = require('../../../server_config/settings.json');
|
||||
|
||||
const s3 = new AWS.S3();
|
||||
|
||||
const resultFileName = 'results.json';
|
||||
const resultScreenshotName = 'screenshot.jpg';
|
||||
const resultsFolderName = 'results';
|
||||
|
||||
|
||||
this.saveResult = function(testResults) {
|
||||
const resultFilePath = path.join(resultsFolderName, testResults.runId, resultFileName);
|
||||
const screenshotFilePath = path.join(resultsFolderName, testResults.runId, resultScreenshotName);
|
||||
|
||||
debug('Starting to save screenshot then results.json file on s3...');
|
||||
|
||||
return saveScreenshotIfExists(testResults, screenshotFilePath)
|
||||
|
||||
.then(function() {
|
||||
debug('Saving results file to s3, destination is %s', resultFilePath);
|
||||
return s3PutObject(resultFilePath, JSON.stringify(testResults, null, 2));
|
||||
});
|
||||
};
|
||||
|
||||
|
||||
this.getResult = function(runId) {
|
||||
const resultFilePath = path.join(resultsFolderName, runId, resultFileName);
|
||||
debug('Reading results (runID = %s) from AWS s3...', runId);
|
||||
return s3GetObject(resultFilePath).then(function(bodyBuffer) {
|
||||
return JSON.parse(bodyBuffer.toString('utf-8'));
|
||||
});
|
||||
};
|
||||
|
||||
|
||||
// If there is a screenshot, save it as screenshot.jpg in the same folder as the results
|
||||
function saveScreenshotIfExists(testResults, imagePath) {
|
||||
var deferred = Q.defer();
|
||||
|
||||
if (testResults.screenshotBuffer) {
|
||||
s3PutObject(imagePath, testResults.screenshotBuffer)
|
||||
|
||||
.fail(function() {
|
||||
debug('Image %s could not be saved on s3. Ignoring.', imagePath);
|
||||
})
|
||||
|
||||
.finally(function() {
|
||||
delete testResults.screenshotBuffer;
|
||||
deferred.resolve();
|
||||
});
|
||||
|
||||
} else {
|
||||
debug('Screenshot not found');
|
||||
deferred.resolve();
|
||||
}
|
||||
|
||||
return deferred.promise;
|
||||
}
|
||||
|
||||
|
||||
this.getScreenshot = function(runId) {
|
||||
const screenshotFilePath = path.join(resultsFolderName, runId, resultScreenshotName);
|
||||
debug('Retrieving screenshot (runID = %s) from s3...', runId);
|
||||
return s3GetObject(screenshotFilePath);
|
||||
};
|
||||
|
||||
|
||||
function s3PutObject(path, body, ignoreError) {
|
||||
var deferred = Q.defer();
|
||||
|
||||
s3.putObject({
|
||||
Bucket: serverSettings.awsHosting.s3.bucket,
|
||||
Key: path,
|
||||
Body: body
|
||||
}, function(err, data) {
|
||||
if (err) {
|
||||
debug('Could not save file %s on s3', path);
|
||||
debug(err);
|
||||
deferred.reject('File saving failed on s3');
|
||||
} else {
|
||||
debug('File %s saved on s3', path);
|
||||
deferred.resolve();
|
||||
}
|
||||
});
|
||||
|
||||
return deferred.promise;
|
||||
}
|
||||
|
||||
|
||||
function s3GetObject(path) {
|
||||
var deferred = Q.defer();
|
||||
|
||||
s3.getObject({
|
||||
Bucket: serverSettings.awsHosting.s3.bucket,
|
||||
Key: path
|
||||
}, function(err, data) {
|
||||
if (err) {
|
||||
debug('Failed retrieving object %s from s3', path);
|
||||
debug(err);
|
||||
deferred.reject(err);
|
||||
} else {
|
||||
debug('Response for %s received from s3...', path);
|
||||
deferred.resolve(data.Body);
|
||||
}
|
||||
});
|
||||
|
||||
return deferred.promise;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
module.exports = ResultsDatastore;
|
||||
@@ -1,132 +0,0 @@
|
||||
var fs = require('fs');
|
||||
var rimraf = require('rimraf');
|
||||
var path = require('path');
|
||||
var Q = require('q');
|
||||
var debug = require('debug')('ylt:resultsDatastore');
|
||||
|
||||
|
||||
function ResultsDatastore() {
|
||||
'use strict';
|
||||
|
||||
var resultFileName = 'results.json';
|
||||
var resultScreenshotName = 'screenshot.jpg';
|
||||
var resultsFolderName = 'results';
|
||||
var resultsDir = path.join(__dirname, '..', '..', '..', resultsFolderName);
|
||||
|
||||
|
||||
this.saveResult = function(testResults) {
|
||||
|
||||
var screenshotFilePath = path.join(resultsDir, testResults.runId, resultScreenshotName);
|
||||
var screenshotAPIPath = '/';
|
||||
|
||||
return createResultFolder(testResults.runId)
|
||||
|
||||
.then(function() {
|
||||
return saveScreenshotIfExists(testResults, screenshotFilePath);
|
||||
})
|
||||
|
||||
.then(function() {
|
||||
|
||||
debug('Saving results to disk...');
|
||||
|
||||
var resultFilePath = path.join(resultsDir, testResults.runId, resultFileName);
|
||||
debug('Destination file is %s', resultFilePath);
|
||||
|
||||
return Q.nfcall(fs.writeFile, resultFilePath, JSON.stringify(testResults, null, 2));
|
||||
});
|
||||
};
|
||||
|
||||
|
||||
this.getResult = function(runId) {
|
||||
|
||||
var resultFilePath = path.join(resultsDir, runId, resultFileName);
|
||||
|
||||
debug('Reading results (runID = %s) from disk...', runId);
|
||||
|
||||
return Q.nfcall(fs.readFile, resultFilePath, {encoding: 'utf8'}).then(function(data) {
|
||||
return JSON.parse(data);
|
||||
});
|
||||
};
|
||||
|
||||
|
||||
/*this.deleteResult = function(runId) {
|
||||
var folder = path.join(resultsDir, runId);
|
||||
|
||||
debug('Deleting results (runID = %s) from disk...', runId);
|
||||
|
||||
return Q.nfcall(rimraf, folder);
|
||||
};*/
|
||||
|
||||
|
||||
// The folder /results/folderName/
|
||||
function createResultFolder(runId) {
|
||||
var folder = path.join(resultsDir, runId);
|
||||
|
||||
debug('Creating the folder %s', runId);
|
||||
|
||||
return createGlobalFolder().then(function() {
|
||||
return Q.nfcall(fs.mkdir, folder);
|
||||
});
|
||||
}
|
||||
|
||||
// The folder /results/
|
||||
function createGlobalFolder() {
|
||||
var deferred = Q.defer();
|
||||
|
||||
// Create the results folder if it doesn't exist
|
||||
fs.exists(resultsDir, function(exists) {
|
||||
if (exists) {
|
||||
deferred.resolve();
|
||||
} else {
|
||||
debug('Creating the global results folder', resultsDir);
|
||||
fs.mkdir(resultsDir, function(err) {
|
||||
if (err) {
|
||||
deferred.reject(err);
|
||||
} else {
|
||||
deferred.resolve();
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
return deferred.promise;
|
||||
}
|
||||
|
||||
// If there is a screenshot, save it as screenshot.jpg in the same folder as the results
|
||||
function saveScreenshotIfExists(testResults, path) {
|
||||
var deferred = Q.defer();
|
||||
|
||||
if (testResults.screenshotBuffer) {
|
||||
|
||||
fs.writeFile(path, testResults.screenshotBuffer, function(err) {
|
||||
if (err) {
|
||||
debug('Could not save final screenshot');
|
||||
debug(err);
|
||||
// But it is OK, we don't need to fail the run
|
||||
deferred.resolve();
|
||||
} else {
|
||||
debug('Final screenshot saved: ' + path);
|
||||
deferred.resolve();
|
||||
}
|
||||
});
|
||||
delete testResults.screenshotBuffer;
|
||||
|
||||
} else {
|
||||
debug('Screenshot not found');
|
||||
deferred.resolve();
|
||||
}
|
||||
|
||||
return deferred.promise;
|
||||
}
|
||||
|
||||
this.getScreenshot = function(runId) {
|
||||
|
||||
var screenshotFilePath = path.join(resultsDir, runId, resultScreenshotName);
|
||||
|
||||
debug('Getting screenshot (runID = %s) from disk...', runId);
|
||||
|
||||
return Q.nfcall(fs.readFile, screenshotFilePath);
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = ResultsDatastore;
|
||||
@@ -1,122 +0,0 @@
|
||||
|
||||
|
||||
function RunsDatastore() {
|
||||
'use strict';
|
||||
|
||||
// NOT PERSISTING RUNS
|
||||
// For the moment, maybe one day
|
||||
var runs = {};
|
||||
|
||||
var STATUS_AWAITING = 'awaiting';
|
||||
var STATUS_RUNNING = 'running';
|
||||
var STATUS_COMPLETE = 'complete';
|
||||
var STATUS_FAILED = 'failed';
|
||||
|
||||
|
||||
this.add = function(run, position) {
|
||||
runs[run.runId] = run;
|
||||
this.updatePosition(run.runId, position);
|
||||
};
|
||||
|
||||
|
||||
this.get = function(runId) {
|
||||
return runs[runId];
|
||||
};
|
||||
|
||||
|
||||
this.updatePosition = function(runId, position) {
|
||||
var run = runs[runId];
|
||||
|
||||
if (position > 0) {
|
||||
run.status = {
|
||||
statusCode: STATUS_AWAITING,
|
||||
position: position
|
||||
};
|
||||
} else {
|
||||
run.status = {
|
||||
statusCode: STATUS_RUNNING
|
||||
};
|
||||
}
|
||||
|
||||
runs[runId] = run;
|
||||
};
|
||||
|
||||
|
||||
// When the test is launched, set the progress bar
|
||||
this.updateRunProgress = function(runId, progress) {
|
||||
var run = runs[runId];
|
||||
|
||||
run.progress = progress;
|
||||
|
||||
runs[runId] = run;
|
||||
};
|
||||
|
||||
|
||||
this.markAsComplete = function(runId) {
|
||||
var run = runs[runId];
|
||||
|
||||
run.status = {
|
||||
statusCode: STATUS_COMPLETE
|
||||
};
|
||||
|
||||
runs[runId] = run;
|
||||
};
|
||||
|
||||
|
||||
this.markAsFailed = function(runId, err) {
|
||||
var run = runs[runId];
|
||||
|
||||
var errorMessage;
|
||||
switch(err) {
|
||||
case '1':
|
||||
errorMessage = "Error 1: unknown error";
|
||||
break;
|
||||
case '252':
|
||||
errorMessage = "Error 252: page timeout in Phantomas";
|
||||
break;
|
||||
case '253':
|
||||
errorMessage = "Error 253: Phantomas config error";
|
||||
break;
|
||||
case '254':
|
||||
errorMessage = "Error 254: page loading failed in PhantomJS";
|
||||
break;
|
||||
case '255':
|
||||
errorMessage = "Error 255: Phantomas error";
|
||||
break;
|
||||
case '1001':
|
||||
errorMessage = "Error 1001: JavaScript profiling failed";
|
||||
break;
|
||||
case '1002':
|
||||
errorMessage = "Error 1002: missing Phantomas metrics";
|
||||
break;
|
||||
case '1003':
|
||||
errorMessage = "Error 1003: Phantomas not returning";
|
||||
break;
|
||||
default:
|
||||
errorMessage = err;
|
||||
}
|
||||
|
||||
run.status = {
|
||||
statusCode: STATUS_FAILED,
|
||||
error: errorMessage
|
||||
};
|
||||
|
||||
runs[runId] = run;
|
||||
};
|
||||
|
||||
|
||||
this.delete = function(runId) {
|
||||
delete runs[runId];
|
||||
};
|
||||
|
||||
|
||||
this.list = function() {
|
||||
var runsArray = [];
|
||||
Object.keys(runs).forEach(function(key) {
|
||||
runsArray.push(runs[key]);
|
||||
});
|
||||
return runsArray;
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = RunsDatastore;
|
||||
@@ -1,91 +0,0 @@
|
||||
var Q = require('q');
|
||||
var debug = require('debug')('ylt:runsQueue');
|
||||
|
||||
|
||||
function RunsQueue() {
|
||||
'use strict';
|
||||
|
||||
var queue = [];
|
||||
var lastTestTimestamp = 0;
|
||||
|
||||
this.push = function(runId) {
|
||||
var deferred = Q.defer();
|
||||
//var startingPosition = queue.length;
|
||||
var startingPosition = 0;
|
||||
|
||||
debug('Adding run %s to the queue, position is %d', runId, startingPosition);
|
||||
|
||||
if (startingPosition === 0) {
|
||||
|
||||
// The queue is empty, let's run immediatly
|
||||
queue.push({
|
||||
runId: runId
|
||||
});
|
||||
|
||||
lastTestTimestamp = Date.now();
|
||||
deferred.resolve();
|
||||
|
||||
} else {
|
||||
|
||||
queue.push({
|
||||
runId: runId,
|
||||
positionChangedCallback: function(position) {
|
||||
deferred.notify(position);
|
||||
},
|
||||
itIsTimeCallback: function() {
|
||||
lastTestTimestamp = Date.now();
|
||||
deferred.resolve();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
var promise = deferred.promise;
|
||||
promise.startingPosition = startingPosition;
|
||||
return promise;
|
||||
};
|
||||
|
||||
|
||||
this.getPosition = function(runId) {
|
||||
// Position 0 means it's a work in progress (a run is removed AFTER it is finished, not before)
|
||||
var position = -1;
|
||||
|
||||
queue.some(function(run, index) {
|
||||
if (run.runId === runId) {
|
||||
position = index;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
});
|
||||
|
||||
return position;
|
||||
};
|
||||
|
||||
|
||||
this.remove = function(runId) {
|
||||
var position = this.getPosition(runId);
|
||||
if (position >= 0) {
|
||||
queue.splice(position, 1);
|
||||
}
|
||||
|
||||
// Update other runs' positions
|
||||
queue.forEach(function(run, index) {
|
||||
if (index === 0 && run.itIsTimeCallback) {
|
||||
run.itIsTimeCallback();
|
||||
} else if (index > 0 && run.positionChangedCallback) {
|
||||
run.positionChangedCallback(index);
|
||||
}
|
||||
});
|
||||
|
||||
};
|
||||
|
||||
this.length = function() {
|
||||
return queue.length;
|
||||
};
|
||||
|
||||
// Returns the number of seconds since the last test was launched
|
||||
this.timeSinceLastTestStarted = function() {
|
||||
return Math.round((Date.now() - lastTestTimestamp) / 1000);
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = RunsQueue;
|
||||
@@ -1,95 +0,0 @@
|
||||
var config = (process.env.IS_TEST) ? require('../../../test/fixtures/settings.json') : require('../../../server_config/settings.json');
|
||||
|
||||
var debug = require('debug')('apiLimitsMiddleware');
|
||||
|
||||
|
||||
var apiLimitsMiddleware = function(req, res, next) {
|
||||
'use strict';
|
||||
|
||||
var ipAddress = req.headers['x-forwarded-for'] || req.connection.remoteAddress;
|
||||
|
||||
debug('Entering API Limits Middleware with IP address %s', ipAddress);
|
||||
|
||||
if (req.path.indexOf('/api/') === 0 && !res.locals.hasApiKey) {
|
||||
|
||||
|
||||
// Monitoring requests
|
||||
if (req.path === '/api/runs' && req.method === 'GET') {
|
||||
next();
|
||||
return;
|
||||
}
|
||||
|
||||
// New tests
|
||||
if (req.path === '/api/runs' && req.method === 'POST') {
|
||||
|
||||
if (!runsTable.accepts(ipAddress)) {
|
||||
// Sorry :/
|
||||
debug('Too many tests launched from IP address %s', ipAddress);
|
||||
res.status(429).send('Too many requests');
|
||||
return;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// Every other calls
|
||||
if (!callsTable.accepts(ipAddress)) {
|
||||
// Sorry :/
|
||||
debug('Too many API requests from IP address %s', ipAddress);
|
||||
res.status(429).send('Too many requests');
|
||||
return;
|
||||
}
|
||||
|
||||
debug('Not blocked by the API limits');
|
||||
// It's ok for the moment
|
||||
}
|
||||
|
||||
next();
|
||||
};
|
||||
|
||||
|
||||
var RecordTable = function(maxPerDay) {
|
||||
var table = {};
|
||||
|
||||
// Check if the user overpassed the limit and save its visit
|
||||
this.accepts = function(ipAddress) {
|
||||
if (table[ipAddress]) {
|
||||
|
||||
this.cleanEntry(ipAddress);
|
||||
|
||||
debug('%d visits in the last 24 hours', table[ipAddress].length);
|
||||
|
||||
if (table[ipAddress].length >= maxPerDay) {
|
||||
return false;
|
||||
} else {
|
||||
table[ipAddress].push(Date.now());
|
||||
}
|
||||
|
||||
} else {
|
||||
table[ipAddress] = [];
|
||||
table[ipAddress].push(Date.now());
|
||||
}
|
||||
|
||||
return true;
|
||||
};
|
||||
|
||||
// Clean the table for this guy
|
||||
this.cleanEntry = function(ipAddress) {
|
||||
table[ipAddress] = table[ipAddress].filter(function(date) {
|
||||
return date > Date.now() - 1000*60*60*24;
|
||||
});
|
||||
};
|
||||
|
||||
// Clean the entire table once in a while
|
||||
this.removeOld = function() {
|
||||
for (var ipAddress in table) {
|
||||
this.cleanEntry(ipAddress);
|
||||
}
|
||||
};
|
||||
|
||||
};
|
||||
|
||||
// Init the records tables
|
||||
var runsTable = new RecordTable(config.maxAnonymousRunsPerDay);
|
||||
var callsTable = new RecordTable(config.maxAnonymousCallsPerDay);
|
||||
|
||||
module.exports = apiLimitsMiddleware;
|
||||
@@ -1,42 +0,0 @@
|
||||
var config = (process.env.IS_TEST) ? require('../../../test/fixtures/settings.json') : require('../../../server_config/settings.json');
|
||||
|
||||
var debug = require('debug')('authMiddleware');
|
||||
|
||||
|
||||
var authMiddleware = function(req, res, next) {
|
||||
'use strict';
|
||||
|
||||
if (req.path.indexOf('/api/') === 0) {
|
||||
|
||||
|
||||
if (req.headers && req.headers['x-api-key']) {
|
||||
|
||||
// Test if it's an authorized key
|
||||
if (isApiKeyValid(req.headers['x-api-key'])) {
|
||||
|
||||
// Come in!
|
||||
debug('Authorized key: %s', req.headers['x-api-key']);
|
||||
res.locals.hasApiKey = true;
|
||||
|
||||
} else {
|
||||
|
||||
// Sorry :/
|
||||
debug('Unauthorized key %s', req.headers['x-api-key']);
|
||||
res.status(401).send('Unauthorized');
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
debug('No authorization key');
|
||||
// It's ok for the moment but you might be blocked by the apiLimitsMiddleware, dude
|
||||
}
|
||||
}
|
||||
|
||||
next();
|
||||
};
|
||||
|
||||
|
||||
function isApiKeyValid(apiKey) {
|
||||
return (config.authorizedKeys[apiKey]) ? true : false;
|
||||
}
|
||||
|
||||
module.exports = authMiddleware;
|
||||
@@ -1,12 +0,0 @@
|
||||
var wwwRedirectMiddleware = function(req, res, next) {
|
||||
'use strict';
|
||||
|
||||
// Redirect www.yellowlab.tools to yellowlab.tools without "www" (for SEO purpose)
|
||||
if(/^www\.yellowlab\.tools/.test(req.headers.host)) {
|
||||
res.redirect(301, req.protocol + '://' + req.headers.host.replace(/^www\./, '') + req.url);
|
||||
} else {
|
||||
next();
|
||||
}
|
||||
};
|
||||
|
||||
module.exports = wwwRedirectMiddleware;
|
||||
@@ -1,6 +1,5 @@
|
||||
var async = require('async');
|
||||
var Q = require('q');
|
||||
var ps = require('ps-node');
|
||||
var path = require('path');
|
||||
var debug = require('debug')('ylt:phantomaswrapper');
|
||||
var phantomas = require('phantomas');
|
||||
|
||||
Reference in New Issue
Block a user