diff --git a/Gruntfile.js b/Gruntfile.js index 7bf74da..3ccabe9 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -143,7 +143,7 @@ module.exports = function(grunt) { options: { reporter: 'spec', }, - src: ['test/core/phantomasWrapperTest.js'] + src: ['test/core/imageOptimizerTest.js'] }, coverage: { options: { diff --git a/front/src/js/controllers/ruleCtrl.js b/front/src/js/controllers/ruleCtrl.js index 2b22bfc..398c17f 100644 --- a/front/src/js/controllers/ruleCtrl.js +++ b/front/src/js/controllers/ruleCtrl.js @@ -7,7 +7,7 @@ ruleCtrl.config(['ChartJsProvider', function (ChartJsProvider) { colours: ['#FF5252', '#FF8A80'], responsive: true }); -}]) +}]); ruleCtrl.controller('RuleCtrl', ['$scope', '$rootScope', '$routeParams', '$location', '$sce', 'Menu', 'Results', 'API', function($scope, $rootScope, $routeParams, $location, $sce, Menu, Results, API) { $scope.runId = $routeParams.runId; diff --git a/front/src/js/directives/offendersDirectives.js b/front/src/js/directives/offendersDirectives.js index e5c0967..974e70f 100644 --- a/front/src/js/directives/offendersDirectives.js +++ b/front/src/js/directives/offendersDirectives.js @@ -879,7 +879,7 @@ } return mega.toFixed(1) + ' MB'; - } + }; }); })(); \ No newline at end of file diff --git a/lib/runner.js b/lib/runner.js index 43ead1e..473e751 100644 --- a/lib/runner.js +++ b/lib/runner.js @@ -3,7 +3,7 @@ var debug = require('debug')('ylt:runner'); var phantomasWrapper = require('./tools/phantomas/phantomasWrapper'); var jsExecutionTransformer = require('./tools/jsExecutionTransformer'); -var weightChecker = require('./tools/weightChecker'); +var weightChecker = require('./tools/weightChecker/weightChecker'); var rulesChecker = require('./rulesChecker'); var scoreCalculator = require('./scoreCalculator'); @@ -20,7 +20,9 @@ var Runner = function(params) { }; // Execute Phantomas first - phantomasWrapper.execute(data).then(function(phantomasResults) { + phantomasWrapper.execute(data) + + .then(function(phantomasResults) { data.toolsResults.phantomas = phantomasResults; // Treat the JS Execution Tree from offenders @@ -29,7 +31,9 @@ var Runner = function(params) { // Redownload every file return weightChecker.recheckAllFiles(data); - }).then(function(data) { + }) + + .then(function(data) { // Rules checker var policies = require('./metadata/policies'); @@ -50,12 +54,16 @@ var Runner = function(params) { return data; - }).then(function(data) { + }) + + .then(function(data) { // Finished! deferred.resolve(data); - }).fail(function(err) { + }) + + .fail(function(err) { debug('Run failed'); debug(err); diff --git a/lib/tools/weightChecker/imageOptimizer.js b/lib/tools/weightChecker/imageOptimizer.js new file mode 100644 index 0000000..cf968df --- /dev/null +++ b/lib/tools/weightChecker/imageOptimizer.js @@ -0,0 +1,121 @@ +var debug = require('debug')('ylt:imageOptimizer'); + +var Q = require('q'); +var Imagemin = require('imagemin'); +var jpegoptim = require('imagemin-jpegoptim'); + +var ImageOptimizer = function() { + + var MAX_JPEG_QUALITY = 85; + + function optimizeImage(entry) { + var deferred = Q.defer(); + + var fileSize = entry.weightCheck.body.length; + debug('Current file size is %d', fileSize); + + if (isJpeg(entry)) { + debug('File is a JPEG'); + + // Starting softly with a lossless compression + return compressJpegLosslessly(entry.weightCheck.body) + + .then(function(newFile) { + var newFileSize = newFile.contents.length; + + debug('JPEG lossless compression complete for %s', entry.url); + + if (newFileSize < fileSize) { + entry.weightCheck.lossless = entry.weightCheck.optimized = newFileSize; + entry.weightCheck.isOptimized = false; + debug('Filesize is %d bytes smaller (-%d%)', fileSize - newFileSize, Math.round((fileSize - newFileSize) * 100 / fileSize)); + } + + + // Now let's compress lossy to MAX_JPEG_QUALITY + return compressJpegLossly(entry.weightCheck.body); + }) + + .then(function(newFile) { + var newFileSize = newFile.contents.length; + + debug('JPEG lossy compression complete for %s', entry.url); + + if (newFileSize < fileSize) { + entry.weightCheck.lossy = newFileSize; + entry.weightCheck.isOptimized = false; + debug('Filesize is %d bytes smaller (-%d%)', fileSize - newFileSize, Math.round((fileSize - newFileSize) * 100 / fileSize)); + + if (newFileSize < entry.weightCheck.lossless) { + entry.weightCheck.optimized = newFileSize; + } + } + + return entry; + }); + + } else { + debug('File type is not an optimizable image'); + deferred.resolve(entry); + } + + return deferred.promise; + } + + function isJpeg(entry) { + return entry.isImage && entry.contentType === 'image/jpeg'; + } + + function compressJpegLosslessly(imageBody) { + var deferred = Q.defer(); + var startTime = Date.now(); + + debug('Starting JPEG lossless compression'); + + new Imagemin() + .src(imageBody) + .use(Imagemin.jpegtran()) + .run(function (err, files) { + if (err) { + deferred.reject(err); + } else { + deferred.resolve(files[0]); + var endTime = Date.now(); + debug('compressJpegLosslessly took %d ms', endTime - startTime); + } + }); + + return deferred.promise; + } + + function compressJpegLossly(imageBody) { + var deferred = Q.defer(); + var startTime = Date.now(); + + debug('Starting JPEG lossly compression'); + + new Imagemin() + .src(imageBody) + .use(jpegoptim({max: MAX_JPEG_QUALITY})) + .run(function (err, files) { + if (err) { + deferred.reject(err); + } else { + deferred.resolve(files[0]); + var endTime = Date.now(); + debug('compressJpegLossly took %d ms', endTime - startTime); + } + }); + + return deferred.promise; + } + + return { + //recompressIfImage: recompressIfImage, + optimizeImage: optimizeImage, + compressJpegLosslessly: compressJpegLosslessly, + compressJpegLossly: compressJpegLossly + }; +}; + +module.exports = new ImageOptimizer(); \ No newline at end of file diff --git a/lib/tools/weightChecker.js b/lib/tools/weightChecker/weightChecker.js similarity index 90% rename from lib/tools/weightChecker.js rename to lib/tools/weightChecker/weightChecker.js index df2b6eb..75e1e5b 100644 --- a/lib/tools/weightChecker.js +++ b/lib/tools/weightChecker/weightChecker.js @@ -5,13 +5,12 @@ */ -var debug = require('debug')('ylt:weightChecker'); - -var request = require('request'); -var http = require('http'); -var async = require('async'); +var debug = require('debug')('ylt:weightChecker'); var Q = require('q'); +var http = require('http'); var zlib = require('zlib'); +var async = require('async'); +var request = require('request'); var WeightChecker = function() { @@ -28,7 +27,18 @@ var WeightChecker = function() { // Transform every request into a download function with a callback when done var redownloadList = requestsList.map(function(entry) { return function(callback) { - redownloadEntry(entry, callback); + + redownloadEntry(entry) + + .then(recompressIfImage) + + .then(function(entry) { + callback(null, entry); + }) + + .fail(function(err) { + callback(err); + }); }; }); @@ -118,26 +128,25 @@ var WeightChecker = function() { } - function redownloadEntry(entry, callback) { + function redownloadEntry(entry) { + var deferred = Q.defer(); function onError(message) { debug('Could not download %s Error: %s', entry.url, message); entry.weightCheck = { message: message }; - setImmediate(function() { - callback(null, entry); - }); + deferred.reject(); } if (entry.method !== 'GET') { onError('only downloading GET'); - return; + return deferred.promise; } if (entry.status !== 200) { onError('only downloading requests with status code 200'); - return; + return deferred.promise; } @@ -167,8 +176,10 @@ var WeightChecker = function() { debug('%s downloaded correctly', entry.url); entry.weightCheck = result; - callback(null, entry); + deferred.resolve(entry); }); + + return deferred.promise; } // Inspired by https://github.com/cvan/fastHAR-api/blob/10cec585/app.js @@ -268,11 +279,22 @@ var WeightChecker = function() { }); } + + function recompressIfImage(entry) { + var deferred = Q.defer(); + + deferred.resolve(entry); + + return deferred.promise; + } + + return { recheckAllFiles: recheckAllFiles, listRequestWeight: listRequestWeight, redownloadEntry: redownloadEntry, - download: download + download: download, + recompressIfImage: recompressIfImage }; }; diff --git a/package.json b/package.json index 24f8d32..03c1a33 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,8 @@ "cors": "^2.6.0", "debug": "~2.2.0", "express": "~4.12.4", + "imagemin": "~3.2.0", + "imagemin-jpegoptim": "~4.0.0", "lwip": "0.0.6", "meow": "^3.1.0", "phantomas": "1.10.2", diff --git a/test/core/imageOptimizerTest.js b/test/core/imageOptimizerTest.js new file mode 100644 index 0000000..d170334 --- /dev/null +++ b/test/core/imageOptimizerTest.js @@ -0,0 +1,79 @@ +var should = require('chai').should(); +var imageOptimizer = require('../../lib/tools/weightChecker/imageOptimizer'); +var fs = require('fs'); +var path = require('path'); + +describe('imageOptimizer', function() { + + it('should optimize a JPEG image losslessly', function(done) { + var fileContent = fs.readFileSync(path.resolve(__dirname, '../fixtures/jpeg-image.jpg')); + + var fileSize = fileContent.length; + + imageOptimizer.compressJpegLosslessly(fileContent).then(function(newFile) { + var newFileSize = newFile.contents.length; + newFileSize.should.be.below(fileSize); + done(); + }).fail(function(err) { + done(err); + }); + }); + + it('should optimize a JPEG image lossly', function(done) { + var fileContent = fs.readFileSync(path.resolve(__dirname, '../fixtures/jpeg-image.jpg')); + + var fileSize = fileContent.length; + + imageOptimizer.compressJpegLossly(fileContent).then(function(newFile) { + var newFileSize = newFile.contents.length; + newFileSize.should.be.below(fileSize); + done(); + }).fail(function(err) { + done(err); + }); + }); + + it('should find the best optimization for a jpeg', function(done) { + var fileContent = fs.readFileSync(path.resolve(__dirname, '../fixtures/jpeg-image.jpg')); + var fileSize = fileContent.length; + + var entry = { + method: 'GET', + url: 'http://localhost:8388/an-image.jpg', + requestHeaders: { + 'User-Agent': 'something', + Referer: 'http://www.google.fr/', + Accept: '*/*', + 'Accept-Encoding': 'gzip, deflate' + }, + status: 200, + isImage: true, + type: 'image', + contentType: 'image/jpeg', + contentLength: 999, + weightCheck: { + body: fileContent, + totalWeight: fileSize + 200, + headersSize: 200, + bodySize: fileSize, + isCompressed: false, + uncompressedSize: fileSize + } + }; + + imageOptimizer.optimizeImage(entry) + + .then(function(newEntry) { + newEntry.weightCheck.should.have.a.property('isOptimized').that.equals(false); + newEntry.weightCheck.should.have.a.property('lossless').that.is.below(fileSize); + newEntry.weightCheck.should.have.a.property('lossy').that.is.below(newEntry.weightCheck.lossless); + + done(); + }) + + .fail(function(err) { + done(err); + }); + }); + +}); diff --git a/test/core/phantomasWrapperTest.js b/test/core/phantomasWrapperTest.js index 28bafc5..ef5cef8 100644 --- a/test/core/phantomasWrapperTest.js +++ b/test/core/phantomasWrapperTest.js @@ -48,7 +48,6 @@ describe('phantomasWrapper', function() { } }).then(function(data) { - console.log(data); done('Error: unwanted success'); }).fail(function(err) { diff --git a/test/core/weightCheckerTest.js b/test/core/weightCheckerTest.js index 61bf429..208ff12 100644 --- a/test/core/weightCheckerTest.js +++ b/test/core/weightCheckerTest.js @@ -1,5 +1,5 @@ var should = require('chai').should(); -var weightChecker = require('../../lib/tools/weightChecker'); +var weightChecker = require('../../lib/tools/weightChecker/weightChecker'); describe('weightChecker', function() { @@ -69,8 +69,9 @@ describe('weightChecker', function() { isJS: true }; - weightChecker.redownloadEntry(entry, function(err, newEntry) { - should.not.exist(err); + weightChecker.redownloadEntry(entry) + + .then(function(newEntry) { newEntry.weightCheck.bodySize.should.equal(93636); newEntry.weightCheck.uncompressedSize.should.equal(newEntry.weightCheck.bodySize); @@ -79,6 +80,10 @@ describe('weightChecker', function() { newEntry.weightCheck.body.should.have.string('1.8.3'); done(); + }) + + .fail(function(err) { + done(err); }); }); @@ -96,12 +101,16 @@ describe('weightChecker', function() { contentLength: 999 }; - weightChecker.redownloadEntry(entry, function(err, newEntry) { - should.not.exist(err); + weightChecker.redownloadEntry(entry) + .then(function(errnewEntry) { newEntry.weightCheck.should.have.a.property('message').that.equals('error while downloading: 404'); done(); + }) + + .fail(function(err) { + done(err); }); }); @@ -119,12 +128,16 @@ describe('weightChecker', function() { contentLength: 999 }; - weightChecker.redownloadEntry(entry, function(err, newEntry) { - should.not.exist(err); + weightChecker.redownloadEntry(entry) + .then(function(newEntry) { newEntry.weightCheck.should.have.a.property('message').that.equals('only downloading requests with status code 200'); done(); + }) + + .fail(function(err) { + done(err); }); }); diff --git a/test/fixtures/jpeg-image.jpg b/test/fixtures/jpeg-image.jpg new file mode 100644 index 0000000..b77fca9 Binary files /dev/null and b/test/fixtures/jpeg-image.jpg differ