diff --git a/lib/screenshotHandler.js b/lib/screenshotHandler.js new file mode 100644 index 0000000..c7f725f --- /dev/null +++ b/lib/screenshotHandler.js @@ -0,0 +1,135 @@ +var debug = require('debug')('ylt:screenshotHandler'); +var lwip = require('lwip'); +var tmp = require('temporary'); +var Q = require('q'); +var fs = require('fs'); +var path = require('path'); + + +var screenshotHandler = function() { + + this.getScreenshotTempFile = function() { + + var screenshotTmpFolder = new tmp.Dir(); + var tmpFilePath = path.join(screenshotTmpFolder.path, 'screenshot.jpg'); + var that = this; + + return { + + getTmpFolder: function() { + return screenshotTmpFolder; + }, + + getTmpFilePath: function() { + return tmpFilePath; + }, + + toThumbnail: function(width) { + return that.optimize(tmpFilePath, width); + }, + + deleteTmpFile: function() { + return that.deleteTmpFileAndFolder(tmpFilePath, screenshotTmpFolder); + } + }; + }; + + + this.optimize = function(imagePath, width) { + var that = this; + + debug('Starting screenshot transformation'); + + return this.openImage(imagePath) + + .then(function(image) { + + return that.resizeImage(image, width); + + }) + + .then(this.toBuffer); + }; + + + this.openImage = function(imagePath) { + var deferred = Q.defer(); + + lwip.open(imagePath, function(err, image){ + if (err) { + debug('Could not open imagePath %s', imagePath); + debug(err); + + deferred.reject(err); + } else { + debug('Image correctly open'); + deferred.resolve(image); + } + }); + + return deferred.promise; + }; + + + this.resizeImage = function(image, newWidth) { + var deferred = Q.defer(); + + var currentWidth = image.width(); + var ratio = newWidth / currentWidth; + + image.scale(ratio, function(err, image){ + if (err) { + debug('Could not resize image'); + debug(err); + + deferred.reject(err); + } else { + debug('Image correctly resized'); + deferred.resolve(image); + } + }); + + return deferred.promise; + }; + + + this.toBuffer = function(image) { + var deferred = Q.defer(); + + image.toBuffer('jpg', {quality: 85}, function(err, buffer){ + if (err) { + debug('Could not save image to buffer'); + debug(err); + + deferred.reject(err); + } else { + debug('Image correctly transformed to buffer'); + deferred.resolve(buffer); + } + }); + + return deferred.promise; + }; + + + this.deleteTmpFileAndFolder = function(tmpFilePath, screenshotTmpFolder) { + var deferred = Q.defer(); + + fs.unlink(tmpFilePath, function (err) { + if (err) { + debug('Screenshot file not found, could not be deleted. But it is not a problem.'); + } else { + debug('Screenshot file deleted.'); + } + + screenshotTmpFolder.rmdir(); + debug('Screenshot temp folder deleted'); + + deferred.resolve(); + }); + + return deferred.promise; + }; +}; + +module.exports = new screenshotHandler(); \ No newline at end of file diff --git a/lib/server/controllers/apiController.js b/lib/server/controllers/apiController.js index 4ab4142..d0e0aa7 100644 --- a/lib/server/controllers/apiController.js +++ b/lib/server/controllers/apiController.js @@ -1,6 +1,8 @@ var debug = require('debug')('ylt:server'); +var Q = require('q'); var ylt = require('../../index'); +var ScreenshotHandler = require('../../screenshotHandler'); var RunsQueue = require('../datastores/runsQueue'); var RunsDatastore = require('../datastores/runsDatastore'); var ResultsDatastore = require('../datastores/resultsDatastore'); @@ -24,10 +26,17 @@ var ApiController = function(app) { params: { url: req.body.url, waitForResponse: req.body.waitForResponse !== false && req.body.waitForResponse !== 'false' && req.body.waitForResponse !== 0, - partialResult: req.body.partialResult || null + partialResult: req.body.partialResult || null, + screenshot: req.body.screenshot !== false && req.body.screenshot !== 'false' && req.body.screenshot !== 0 } }; + // Create a temporary folder to save the screenshot + var screenshot; + if (run.params.screenshot) { + screenshot = ScreenshotHandler.getScreenshotTempFile(); + } + // Add test to the testQueue debug('Adding test %s to the queue', run.runId); var queuePromise = queue.push(run.runId); @@ -49,81 +58,121 @@ var ApiController = function(app) { debug('Launching test %s on %s', run.runId, run.params.url); - ylt(run.params.url) + var runOptions = { + screenshot: run.params.screenshot ? screenshot.getTmpFilePath() : false + }; - .then(function(data) { + return ylt(run.params.url, runOptions); - debug('Success'); - + }) + // Phantomas completed, let's save the screenshot if any + .then(function(data) { - // Save result in datastore - data.runId = run.runId; - resultsDatastore.saveResult(data) - .then(function() { + debug('Success'); + data.runId = run.runId; - runsDatastore.markAsComplete(run.runId); - - // Send result if the user was waiting - if (run.params.waitForResponse) { + + // Some conditional steps are made if there is a screenshot + var screenshotPromise = Q.resolve(); - // If the user only wants a portion of the result (partialResult option) - switch(run.params.partialResult) { - case 'generalScores': - res.redirect(302, '/api/results/' + run.runId + '/generalScores'); - break; - case 'rules': - res.redirect(302, '/api/results/' + run.runId + '/rules'); - break; - case 'javascriptExecutionTree': - res.redirect(302, '/api/results/' + run.runId + '/javascriptExecutionTree'); - break; - case 'phantomas': - res.redirect(302, '/api/results/' + run.runId + '/toolsResults/phantomas'); - break; - default: - res.redirect(302, '/api/results/' + run.runId); - } - } - - }) - .fail(function(err) { - debug('Saving results to resultsDatastore failed:'); - debug(err); + if (run.params.screenshot) { + + // Replace the empty promise created earlier with Q.resolve() + screenshotPromise = screenshot.toThumbnail(640) + + // Read screenshot + .then(function(screenshotBuffer) { + + if (screenshotBuffer) { + debug('Image optimized'); + data.screenshotBuffer = screenshotBuffer; - res.status(500).send('Saving results failed'); - }); + // Official path to get the image + data.screenshotUrl = '/result/' + data.runId + '/screenshot.jpg'; + } + delete data.params.options.screenshot; + + }) + // Delete screenshot temporary file + .then(screenshot.deleteTmpFile); + + } + + // Let's continue + screenshotPromise + + // Save results + .then(function() { + return resultsDatastore.saveResult(data); }) + // Mark as the run as complete and send the response if the request is still waiting + .then(function() { + + debug('Result saved in datastore'); + + runsDatastore.markAsComplete(run.runId); + + if (run.params.waitForResponse) { + + // If the user only wants a portion of the result (partialResult option) + switch(run.params.partialResult) { + case 'generalScores': + res.redirect(302, '/api/results/' + run.runId + '/generalScores'); + break; + case 'rules': + res.redirect(302, '/api/results/' + run.runId + '/rules'); + break; + case 'javascriptExecutionTree': + res.redirect(302, '/api/results/' + run.runId + '/javascriptExecutionTree'); + break; + case 'phantomas': + res.redirect(302, '/api/results/' + run.runId + '/toolsResults/phantomas'); + break; + default: + res.redirect(302, '/api/results/' + run.runId); + } + } + + }) .fail(function(err) { - console.error('Test failed for URL: %s', run.params.url); console.error(err.toString()); runsDatastore.markAsFailed(run.runId, err.toString()); - res.status(400).send('Bad request'); - - }) - - .finally(function() { - queue.remove(run.runId); + res.status(500).send('An error occured'); }); - }).fail(function(err) { - console.error('Error or YLT\'s core instanciation'); - console.error(err); - console.error(err.stack); + }) + + .fail(function(err) { + + console.error('Test failed for URL: %s', run.params.url); + console.error(err.toString()); + + runsDatastore.markAsFailed(run.runId, err.toString()); + + res.status(400).send('Bad request'); + + }) + + .finally(function() { + queue.remove(run.runId); }); - // The user doesn't not want to wait for the response, sending the run ID only + + // The user doesn't want to wait for the response, sending the run ID only if (!run.params.waitForResponse) { console.log('Sending response without waiting.'); res.setHeader('Content-Type', 'application/json'); res.send(JSON.stringify({runId: run.runId})); } + }); + // Retrive one run by id app.get('/api/runs/:id', function(req, res) { var runId = req.params.id; diff --git a/lib/server/datastores/resultsDatastore.js b/lib/server/datastores/resultsDatastore.js index 3392337..05cf617 100644 --- a/lib/server/datastores/resultsDatastore.js +++ b/lib/server/datastores/resultsDatastore.js @@ -9,24 +9,28 @@ function ResultsDatastore() { 'use strict'; var resultFileName = 'results.json'; + var resultScreenshotName = 'screenshot.jpg'; var resultsFolderName = 'results'; var resultsDir = path.join(__dirname, '..', '..', '..', resultsFolderName); this.saveResult = function(testResults) { - var promise = createResultFolder(testResults.runId); + + return createResultFolder(testResults.runId) - debug('Saving results to disk...'); + .then(function() { + return saveScreenshotIfExists(testResults); + }) - promise.then(function() { + .then(function() { - var resultFilePath = path.join(resultsDir, testResults.runId, resultFileName); - debug('Destination file is %s', resultFilePath); - - return Q.nfcall(fs.writeFile, resultFilePath, JSON.stringify(testResults, null, 2)); - }); + debug('Saving results to disk...'); - return promise; + var resultFilePath = path.join(resultsDir, testResults.runId, resultFileName); + debug('Destination file is %s', resultFilePath); + + return Q.nfcall(fs.writeFile, resultFilePath, JSON.stringify(testResults, null, 2)); + }); }; @@ -84,6 +88,24 @@ function ResultsDatastore() { return deferred.promise; } + + // If there is a screenshot, save it as screenshot.jpg in the same folder as the results + function saveScreenshotIfExists(testResults) { + var deferred = Q.defer(); + + if (testResults.screenshotBuffer) { + + var screenshotFilePath = path.join(resultsDir, testResults.runId, resultScreenshotName); + fs.writeFile(screenshotFilePath, testResults.screenshotBuffer); + + delete testResults.screenshotBuffer; + + } else { + deferred.resolve(); + } + + return deferred; + } } module.exports = ResultsDatastore; \ No newline at end of file diff --git a/lib/server/datastores/runsQueue.js b/lib/server/datastores/runsQueue.js index 26a6717..e99f1fb 100644 --- a/lib/server/datastores/runsQueue.js +++ b/lib/server/datastores/runsQueue.js @@ -1,4 +1,5 @@ var Q = require('q'); +var debug = require('debug')('ylt:runsQueue'); function RunsQueue() { @@ -11,6 +12,8 @@ function RunsQueue() { var deferred = Q.defer(); var startingPosition = queue.length; + debug('Adding run %s to the queue, position is %d', runId, startingPosition); + if (startingPosition === 0) { // The queue is empty, let's run immediatly diff --git a/lib/tools/phantomas/phantomasWrapper.js b/lib/tools/phantomas/phantomasWrapper.js index 594cde9..c24dd67 100644 --- a/lib/tools/phantomas/phantomasWrapper.js +++ b/lib/tools/phantomas/phantomasWrapper.js @@ -1,9 +1,9 @@ -var async = require('async'); -var Q = require('q'); -var ps = require('ps-node'); -var path = require('path'); -var debug = require('debug')('ylt:phantomaswrapper'); -var phantomas = require('phantomas'); +var async = require('async'); +var Q = require('q'); +var ps = require('ps-node'); +var path = require('path'); +var debug = require('debug')('ylt:phantomaswrapper'); +var phantomas = require('phantomas'); var PhantomasWrapper = function() { @@ -20,17 +20,18 @@ var PhantomasWrapper = function() { this.execute = function(data) { var deferred = Q.defer(); - var task = data.params; + var options = { // Cusomizable options - timeout: task.options.timeout || 60, + 'timeout': task.options.timeout || 60, 'js-deep-analysis': task.options.jsDeepAnalysis || false, 'user-agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.110 Safari/537.36', + 'screenshot': task.options.screenshot || false, // Mandatory - reporter: 'json:pretty', + 'reporter': 'json:pretty', 'analyze-css': true, 'skip-modules': [ 'blockDomains', // not needed @@ -44,7 +45,6 @@ var PhantomasWrapper = function() { 'jQuery', // overridden 'jserrors', // overridden 'pageSource', // not needed - 'screenshot', // not needed for the moment 'waitForSelector', // not needed 'windowPerformance' // overriden ].join(','), @@ -100,7 +100,7 @@ var PhantomasWrapper = function() { debug('Returning from Phantomas'); // Adding some YellowLabTools errors here - if (json && json.metrics && !json.metrics.javascriptExecutionTree) { + if (json && json.metrics && (!json.metrics.javascriptExecutionTree || !json.offenders.javascriptExecutionTree)) { err = 1001; } @@ -130,10 +130,11 @@ var PhantomasWrapper = function() { if (err) { debug('All ' + triesNumber + ' attemps failed for the test'); deferred.reject(err); + } else { - // Success!!! deferred.resolve(json); + } }); diff --git a/package.json b/package.json index e2d31ec..264a7f1 100644 --- a/package.json +++ b/package.json @@ -16,10 +16,12 @@ "cors": "^2.5.2", "debug": "~2.1.0", "express": "~4.10.6", + "lwip": "0.0.6", "phantomas": "1.9.0", "ps-node": "0.0.3", "q": "~1.1.2", - "rimraf": "~2.2.8" + "rimraf": "~2.2.8", + "temporary": "0.0.8" }, "devDependencies": { "chai": "^1.10.0", diff --git a/test/api/apiTest.js b/test/api/apiTest.js index bb20128..50db2ae 100644 --- a/test/api/apiTest.js +++ b/test/api/apiTest.js @@ -131,7 +131,6 @@ describe('api', function() { }, function(error, response, body) { if (!error && response.statusCode === 302) { - console.log(response.headers.location); response.headers.should.have.a.property('location').that.is.a('string'); response.headers.location.should.contain('/rules'); @@ -160,6 +159,14 @@ describe('api', function() { body.should.have.a.property('toolsResults').that.is.an('object'); body.should.have.a.property('javascriptExecutionTree').that.is.an('object'); + // Check if the screenshot temporary file was correctly removed + body.params.options.should.not.have.a.property('screenshot'); + // Check if the screenshot buffer was correctly removed + body.should.not.have.a.property('screenshotBuffer'); + // Check if the screenshot url is here + body.should.have.a.property('screenshotUrl'); + body.screenshotUrl.should.have.string('/result/' + body.runId + '/screenshot.jpg'); + done(); } else { diff --git a/test/api/screenshotHandlerTest.js b/test/api/screenshotHandlerTest.js new file mode 100644 index 0000000..54127f3 --- /dev/null +++ b/test/api/screenshotHandlerTest.js @@ -0,0 +1,126 @@ +var should = require('chai').should(); +var ScreenshotHandler = require('../../lib/screenshotHandler'); + +var fs = require('fs'); +var path = require('path'); + +describe('screenshotHandler', function() { + + var imagePath = path.join(__dirname, '../fixtures/logo-large.png'); + var screenshot, lwipImage; + + + it('should open an image and return an lwip object', function(done) { + ScreenshotHandler.openImage(imagePath) + .then(function(image) { + lwipImage = image; + + lwipImage.should.be.an('object'); + lwipImage.width().should.equal(620); + lwipImage.height().should.equal(104); + + done(); + }) + .fail(function(err) { + done(err); + }); + }); + + + it('should resize an lwip image', function(done) { + ScreenshotHandler.resizeImage(lwipImage, 310) + .then(function(image) { + lwipImage = image; + + lwipImage.width().should.equal(310); + lwipImage.height().should.equal(52); + + done(); + }) + .fail(function(err) { + done(err); + }); + }); + + + it('should transform a lwip image into a buffer', function(done) { + ScreenshotHandler.toBuffer(lwipImage) + .then(function(buffer) { + buffer.should.be.an.instanceof(Buffer); + done(); + }) + .fail(function(err) { + done(err); + }); + }); + + + it('should optimize an image and return a buffered version', function(done) { + ScreenshotHandler.optimize(imagePath, 200) + .then(function(buffer) { + buffer.should.be.an.instanceof(Buffer); + done(); + }) + .fail(function(err) { + done(err); + }); + }); + + + it('should provide a temporary file object', function() { + screenshot = ScreenshotHandler.getScreenshotTempFile(); + + screenshot.should.have.a.property('getTmpFolder').that.is.a('function'); + screenshot.should.have.a.property('getTmpFilePath').that.is.a('function'); + screenshot.should.have.a.property('toThumbnail').that.is.a('function'); + screenshot.should.have.a.property('deleteTmpFile').that.is.a('function'); + }); + + + it('should have created the temporary folder', function() { + var folder = screenshot.getTmpFolder(); + fs.existsSync(folder.path).should.equal(true); + }); + + + it('should respond a temporary file', function() { + var file = screenshot.getTmpFilePath(); + file.should.have.string('/screenshot.jpg'); + }); + + + it('should delete the temp folder when there is no file', function(done) { + var tmpFolderPath = screenshot; + + screenshot.deleteTmpFile() + .then(function() { + fs.existsSync(screenshot.getTmpFolder().path).should.equal(false); + done(); + }) + .fail(function(err) { + done(err); + }); + }); + + it('should delete the temp folder with the screenshot inside', function(done) { + screenshot = ScreenshotHandler.getScreenshotTempFile(); + var tmpFolderPath = screenshot.getTmpFolder().path; + var tmpImagePath = path.join(tmpFolderPath, 'screenshot.jpg'); + + // Copy image + var testImage = fs.readFileSync(imagePath); + fs.writeFileSync(tmpImagePath, testImage); + + fs.existsSync(tmpImagePath).should.equal(true); + + screenshot.deleteTmpFile() + .then(function() { + fs.existsSync(tmpImagePath).should.equal(false); + fs.existsSync(tmpFolderPath).should.equal(false); + done(); + }) + .fail(function(err) { + done(err); + }); + }); +}); \ No newline at end of file diff --git a/test/fixtures/logo-large.png b/test/fixtures/logo-large.png new file mode 100644 index 0000000..f739504 Binary files /dev/null and b/test/fixtures/logo-large.png differ