From ac915e89733230369a63cea5c1e43fcf0f04bdfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABl=20M=C3=A9tais?= Date: Sun, 4 Oct 2020 02:43:10 +0200 Subject: [PATCH] Fix screenshots and continue implementing phantomas v2 --- lib/screenshotHandler.js | 73 ++++++++++++----------- lib/server/controllers/apiController.js | 29 ++++----- lib/server/datastores/resultsDatastore.js | 35 +++++++---- lib/tools/phantomas/phantomasWrapper.js | 52 ++++++++++++++-- server_config/settings-dareboost.json | 19 ++++++ server_config/settings-prod.json | 12 ++-- server_config/settings.json | 6 +- test/api/screenshotHandlerTest.js | 70 ++++------------------ 8 files changed, 161 insertions(+), 135 deletions(-) create mode 100644 server_config/settings-dareboost.json diff --git a/lib/screenshotHandler.js b/lib/screenshotHandler.js index b960672..6c9629f 100644 --- a/lib/screenshotHandler.js +++ b/lib/screenshotHandler.js @@ -1,6 +1,5 @@ var debug = require('debug')('ylt:screenshotHandler'); var Jimp = require('jimp'); -var tmp = require('temporary'); var Q = require('q'); var fs = require('fs'); var path = require('path'); @@ -8,44 +7,22 @@ var path = require('path'); var screenshotHandler = function() { - this.getScreenshotTempFile = function() { - - var screenshotTmpFolder = new tmp.Dir(); - var tmpFilePath = path.join(screenshotTmpFolder.path, 'screenshot.png'); - var that = this; - - return { - - getTmpFolder: function() { - return screenshotTmpFolder; - }, - - getTmpFilePath: function() { - return tmpFilePath; - }, - - toThumbnail: function(width) { - return that.optimize(tmpFilePath, width); - }, - - deleteTmpFile: function() { - return that.deleteTmpFileAndFolder(tmpFilePath, screenshotTmpFolder); - } - }; - }; + var tmpFolderPath = 'tmp'; + var tmpFolderFullPath = path.join(__dirname, '..', tmpFolderPath); + var tmpFileName = 'temp-screenshot.png'; + var tmpFileFullPath = path.join(tmpFolderFullPath, tmpFileName); - this.optimize = function(imagePath, width) { + this.findAndOptimizeScreenshot = function(width) { var that = this; debug('Starting screenshot transformation'); - return this.openImage(imagePath) + return this.openImage(tmpFileFullPath) .then(function(image) { - + that.deleteTmpFile(tmpFileFullPath); return that.resizeImage(image, width); - }) .then(this.addWhiteBackground) @@ -147,24 +124,48 @@ var screenshotHandler = function() { }; - this.deleteTmpFileAndFolder = function(tmpFilePath, screenshotTmpFolder) { + this.deleteTmpFile = function(tmpFilePath) { var deferred = Q.defer(); fs.unlink(tmpFilePath, function (err) { if (err) { - debug('Screenshot file not found, could not be deleted. But it is not a problem.'); + debug('Screenshot temporary file not found, could not be deleted. But it is not a problem.'); } else { - debug('Screenshot file deleted.'); + debug('Screenshot temporary file deleted.'); } - screenshotTmpFolder.rmdir(); - debug('Screenshot temp folder deleted'); - deferred.resolve(); }); return deferred.promise; }; + + // Create a /tmp folder on the project's root directory + this.createTmpScreenshotFolder = function() { + var deferred = Q.defer(); + + // Create the folder if it doesn't exist + fs.exists(tmpFolderFullPath, function(exists) { + if (exists) { + deferred.resolve(); + } else { + debug('Creating the tmp image folder', tmpFolderFullPath); + fs.mkdir(tmpFolderFullPath, function(err) { + if (err) { + deferred.reject(err); + } else { + deferred.resolve(); + } + }); + } + }); + + return deferred.promise; + }; + + this.getTmpFileRelativePath = function() { + return tmpFolderPath + '/' + tmpFileName; + }; }; module.exports = new screenshotHandler(); diff --git a/lib/server/controllers/apiController.js b/lib/server/controllers/apiController.js index 4c7c245..84d1b88 100644 --- a/lib/server/controllers/apiController.js +++ b/lib/server/controllers/apiController.js @@ -51,11 +51,8 @@ var ApiController = function(app) { } }; - // Create a temporary folder to save the screenshot - var screenshot; - if (run.params.screenshot) { - screenshot = ScreenshotHandler.getScreenshotTempFile(); - } + // Create the tmp folder if it doesn't exist + ScreenshotHandler.createTmpScreenshotFolder(run.runId); // Add test to the testQueue debug('Adding test %s to the queue', run.runId); @@ -78,7 +75,7 @@ var ApiController = function(app) { console.log('Launching test ' + run.runId + ' on ' + run.params.url); var runOptions = { - screenshot: run.params.screenshot ? screenshot.getTmpFilePath() : false, + screenshot: run.params.screenshot ? ScreenshotHandler.getTmpFileRelativePath() : false, device: run.params.device, proxy: run.params.proxy, waitForSelector: run.params.waitForSelector, @@ -95,40 +92,36 @@ var ApiController = function(app) { }) - // Phantomas completed, let's save the screenshot if any + // Phantomas completed .then(function(data) { debug('Success'); data.runId = run.runId; - // Some conditional steps are made if there is a screenshot + // Some conditional steps exist if there is a screenshot var screenshotPromise = Q.resolve(); if (run.params.screenshot) { - + + var screenshotSize = serverSettings.screenshotWidth ? serverSettings.screenshotWidth[run.params.device] : 400; + // Replace the empty promise created earlier with Q.resolve() - screenshotPromise = screenshot.toThumbnail(serverSettings.screenshotWidth || 400) + screenshotPromise = ScreenshotHandler.findAndOptimizeScreenshot(screenshotSize) // Read screenshot .then(function(screenshotBuffer) { - if (screenshotBuffer) { debug('Image optimized'); data.screenshotBuffer = screenshotBuffer; - - // Official path to get the image - data.screenshotUrl = 'api/results/' + data.runId + '/screenshot.jpg'; + data.screenshotUrl = '/api/results/' + data.runId + '/screenshot.jpg' } - }) - - // Delete screenshot temporary file - .then(screenshot.deleteTmpFile) // Don't worry if there's an error .fail(function(err) { debug('An error occured while creating the screenshot\'s thumbnail. Ignoring and continuing...'); + debug(err); }); } diff --git a/lib/server/datastores/resultsDatastore.js b/lib/server/datastores/resultsDatastore.js index a8119f9..262f53b 100644 --- a/lib/server/datastores/resultsDatastore.js +++ b/lib/server/datastores/resultsDatastore.js @@ -16,10 +16,13 @@ function ResultsDatastore() { this.saveResult = function(testResults) { + var screenshotFilePath = path.join(resultsDir, testResults.runId, resultScreenshotName); + var screenshotAPIPath = '/' + return createResultFolder(testResults.runId) .then(function() { - return saveScreenshotIfExists(testResults); + return saveScreenshotIfExists(testResults, screenshotFilePath); }) .then(function() { @@ -56,10 +59,10 @@ function ResultsDatastore() { // The folder /results/folderName/ - function createResultFolder(folderName) { - var folder = path.join(resultsDir, folderName); + function createResultFolder(runId) { + var folder = path.join(resultsDir, runId); - debug('Creating the folder %s', folderName); + debug('Creating the folder %s', runId); return createGlobalFolder().then(function() { return Q.nfcall(fs.mkdir, folder); @@ -89,22 +92,32 @@ function ResultsDatastore() { return deferred.promise; } + this.getResultFolder = function(runId) { + return path.join(resultsDir, runId); + }; + // If there is a screenshot, save it as screenshot.jpg in the same folder as the results - function saveScreenshotIfExists(testResults) { + function saveScreenshotIfExists(testResults, path) { var deferred = Q.defer(); if (testResults.screenshotBuffer) { - var screenshotFilePath = path.join(resultsDir, testResults.runId, resultScreenshotName); - fs.writeFile(screenshotFilePath, testResults.screenshotBuffer); - + fs.writeFile(path, testResults.screenshotBuffer, function(err) { + if (err) { + debug('Could not save final screenshot'); + debug(err); + // But it is OK, we don't need to fail the run + deferred.resolve(); + } else { + debug('Final screenshot saved: ' + path); + deferred.resolve(); + } + }); delete testResults.screenshotBuffer; - } else { - deferred.resolve(); } - return deferred; + return deferred.promise; } this.getScreenshot = function(runId) { diff --git a/lib/tools/phantomas/phantomasWrapper.js b/lib/tools/phantomas/phantomasWrapper.js index 636408e..51b899d 100644 --- a/lib/tools/phantomas/phantomasWrapper.js +++ b/lib/tools/phantomas/phantomasWrapper.js @@ -17,7 +17,7 @@ var PhantomasWrapper = function() { var deferred = Q.defer(); var task = data.params; - var options = { + /*var options = { // Cusomizable options 'engine': task.options.phantomasEngine || 'webkit', @@ -59,6 +59,51 @@ var PhantomasWrapper = function() { ].join(',') }; + */ + + var options = { + + // Cusomizable options + 'timeout': task.options.timeout || 60, + 'user-agent': (task.options.device === 'desktop') ? 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) YLT Chrome/85.0.4183.121 Safari/537.36' : null, + 'tablet': (task.options.device === 'tablet'), + 'phone': (task.options.device === 'phone'), + 'screenshot': task.options.screenshot || false, + 'viewport': (task.options.device === 'desktop') ? '1280x800x1' : null, + 'wait-for-network-idle': true, + //'wait-for-selector': task.options.waitForSelector, + 'cookie': task.options.cookie, + 'auth-user': task.options.authUser, + 'auth-pass': task.options.authPass, + 'block-domain': task.options.blockDomain, + 'allow-domain': task.options.allowDomain, + 'no-externals': task.options.noExternals, + + // Mandatory + 'analyze-css': true, + 'ignore-ssl-errors': true, + /*'skip-modules': [ + 'ajaxRequests', // overridden + 'domHiddenContent', // overridden + 'domMutations', // not compatible with webkit + 'domQueries', // overridden + 'events', // overridden + 'filmStrip', // not needed + 'har', // not needed for the moment + 'javaScriptBottlenecks', // needs to be launched after custom module scopeYLT + 'jQuery', // overridden + 'jserrors', // overridden + 'lazyLoadableImages', //overridden + 'pageSource', // not needed + 'windowPerformance' // overridden + ].join(','),*/ + /*'include-dirs': [ + path.join(__dirname, 'custom_modules/core'), + path.join(__dirname, 'custom_modules/modules') + ].join(',')*/ + }; + + // Proxy option can't be set to null or undefined... // this is why it's set now and not in the object above if (task.options.proxy) { @@ -67,10 +112,7 @@ var PhantomasWrapper = function() { // It's time to launch the test!!! - const promise = phantomas(task.url, { - 'analyze-css': true, - 'screenshot': true - }); + const promise = phantomas(task.url, options); // handle the promise promise. diff --git a/server_config/settings-dareboost.json b/server_config/settings-dareboost.json new file mode 100644 index 0000000..c32d5a4 --- /dev/null +++ b/server_config/settings-dareboost.json @@ -0,0 +1,19 @@ +{ + "serverPort": 80, + "phantomasEngine": "webkit", + "googleAnalyticsId": "", + "screenshotWidth": 400, + "baseUrl": "/", + "authorizedKeys": { + + }, + "maxAnonymousRunsPerDay": 99999999, + "maxAnonymousCallsPerDay": 99999999, + "blockedUrls": [], + + "sponsoring" : { + "home": "Generously hosted by



and sponsored by Dareboost, the website speed test service
", "dashboard": null, + "dashboard": "
Test your website for free and get additional insights with our sponsor: dareboost.com
", + "about": "


This public instance of Yellow Lab Tools is generously hosted by



Yellow Lab Tools is also made possible thanks to our sponsor
" + } +} \ No newline at end of file diff --git a/server_config/settings-prod.json b/server_config/settings-prod.json index aff5cc0..272df56 100644 --- a/server_config/settings-prod.json +++ b/server_config/settings-prod.json @@ -2,7 +2,11 @@ "serverPort": 80, "phantomasEngine": "webkit", "googleAnalyticsId": "", - "screenshotWidth": 400, + "screenshotWidth": { + "desktop": 600, + "tablet": 420, + "phone": 360 + }, "baseUrl": "/", "authorizedKeys": { @@ -12,8 +16,8 @@ "blockedUrls": [], "sponsoring" : { - "home": "(this is a private instance)", - "dashboard": null, - "about": "(this is a private instance)" + "home": "Generously hosted by



and sponsored by Dareboost, the website speed test service
", "dashboard": null, + "dashboard": "
Test your website for free and get additional insights with our sponsor: dareboost.com
", + "about": "


This public instance of Yellow Lab Tools is generously hosted by



Yellow Lab Tools is also made possible thanks to our sponsor
" } } \ No newline at end of file diff --git a/server_config/settings.json b/server_config/settings.json index a23c7ff..5eabca0 100644 --- a/server_config/settings.json +++ b/server_config/settings.json @@ -2,7 +2,11 @@ "serverPort": 8383, "phantomasEngine": "webkit", "googleAnalyticsId": "", - "screenshotWidth": 400, + "screenshotWidth": { + "desktop": 600, + "tablet": 420, + "phone": 360 + }, "baseUrl": "/", "authorizedKeys": { diff --git a/test/api/screenshotHandlerTest.js b/test/api/screenshotHandlerTest.js index 267a44e..82a77bd 100644 --- a/test/api/screenshotHandlerTest.js +++ b/test/api/screenshotHandlerTest.js @@ -3,6 +3,7 @@ var ScreenshotHandler = require('../../lib/screenshotHandler'); var fs = require('fs'); var path = require('path'); +var rimraf = require('rimraf'); describe('screenshotHandler', function() { @@ -55,10 +56,14 @@ describe('screenshotHandler', function() { }); - it('should optimize an image and return a buffered version', function(done) { - ScreenshotHandler.optimize(imagePath, 200) + it('should create the tmp folder if it doesn\'t exist', function(done) { + // Delete tmp folder if it exists + rimraf.sync("/some/directory"); + + // The function we want to test + ScreenshotHandler.createTmpScreenshotFolder() .then(function(buffer) { - buffer.should.be.an.instanceof(Buffer); + fs.existsSync(path.join(__dirname, '../../tmp')).should.equal(true); done(); }) .fail(function(err) { @@ -66,63 +71,8 @@ describe('screenshotHandler', function() { }); }); - - it('should provide a temporary file object', function() { - screenshot = ScreenshotHandler.getScreenshotTempFile(); - - screenshot.should.have.a.property('getTmpFolder').that.is.a('function'); - screenshot.should.have.a.property('getTmpFilePath').that.is.a('function'); - screenshot.should.have.a.property('toThumbnail').that.is.a('function'); - screenshot.should.have.a.property('deleteTmpFile').that.is.a('function'); + it('should return the tmp folder path', function() { + ScreenshotHandler.getTmpFileRelativePath().should.equal('tmp/temp-screenshot.png'); }); - - it('should have created the temporary folder', function() { - var folder = screenshot.getTmpFolder(); - fs.existsSync(folder.path).should.equal(true); - }); - - - it('should respond a temporary file', function() { - var file = screenshot.getTmpFilePath(); - file.should.have.string('/screenshot.png'); - }); - - - it('should delete the temp folder when there is no file', function(done) { - var tmpFolderPath = screenshot; - - screenshot.deleteTmpFile() - .delay(1000) - .then(function() { - fs.existsSync(screenshot.getTmpFolder().path).should.equal(false); - done(); - }) - .fail(function(err) { - done(err); - }); - }); - - it('should delete the temp folder with the screenshot inside', function(done) { - screenshot = ScreenshotHandler.getScreenshotTempFile(); - var tmpFolderPath = screenshot.getTmpFolder().path; - var tmpImagePath = path.join(tmpFolderPath, 'screenshot.png'); - - // Copy image - var testImage = fs.readFileSync(imagePath); - fs.writeFileSync(tmpImagePath, testImage); - - fs.existsSync(tmpImagePath).should.equal(true); - - screenshot.deleteTmpFile() - .delay(1000) - .then(function() { - fs.existsSync(tmpImagePath).should.equal(false); - fs.existsSync(tmpFolderPath).should.equal(false); - done(); - }) - .fail(function(err) { - done(err); - }); - }); });