diff --git a/lib/tools/weightChecker/fileMinifier.js b/lib/tools/weightChecker/fileMinifier.js index e80d2e8..8e0279e 100644 --- a/lib/tools/weightChecker/fileMinifier.js +++ b/lib/tools/weightChecker/fileMinifier.js @@ -20,12 +20,12 @@ var FileMinifier = function() { var fileSize = entry.weightCheck.uncompressedSize; debug('Let\'s try to optimize %s', entry.url); debug('Current file size is %d', fileSize); + var startTime = Date.now(); - if (entry.isJS) { + if (entry.isJS && !isKnownAsMinified(entry.url) && !looksAlreadyMinified(entry.weightCheck.body)) { debug('File is a JS'); - // Starting softly with a lossless compression return minifyJs(entry.weightCheck.body) .then(function(newFile) { @@ -34,6 +34,8 @@ var FileMinifier = function() { return entry; } + var endTime = Date.now(); + var newFileSize = newFile.length; debug('JS minification complete for %s', entry.url); @@ -56,7 +58,6 @@ var FileMinifier = function() { debug('File is a CSS'); - // Starting softly with a lossless compression return minifyCss(entry.weightCheck.body) .then(function(newFile) { @@ -65,6 +66,9 @@ var FileMinifier = function() { return entry; } + var endTime = Date.now(); + debug('CSS minification took %dms', endTime - startTime); + var newFileSize = newFile.length; debug('CSS minification complete for %s', entry.url); @@ -87,7 +91,6 @@ var FileMinifier = function() { debug('File is an HTML'); - // Starting softly with a lossless compression return minifyHtml(entry.weightCheck.body) .then(function(newFile) { @@ -96,6 +99,9 @@ var FileMinifier = function() { return entry; } + var endTime = Date.now(); + debug('HTML minification took %dms', endTime - startTime); + var newFileSize = newFile.length; debug('HTML minification complete for %s', entry.url); @@ -132,18 +138,118 @@ var FileMinifier = function() { // Uglify function minifyJs(body) { + + // Splitting the Uglify function because it sometime takes too long (more than 10 seconds) + // I hope that, by splitting, it can be a little more asynchronous, so the application doesn't freeze. + + return splittedUglifyStep1(body) + .delay(1) + .then(splittedUglifyStep2) + .delay(1) + .then(splittedUglifyStep3) + .delay(1) + .then(splittedUglifyStep4) + .delay(1) + .then(splittedUglifyStep5) + .delay(1) + .then(splittedUglifyStep6) + .delay(1) + .then(splittedUglifyStep7); + + } + + function splittedUglifyStep1(code) { var deferred = Q.defer(); + var startTime = Date.now(); try { - var result = UglifyJS.minify(body, {fromString: true}); - deferred.resolve(result.code); + var toplevel_ast = UglifyJS.parse(code); + + var endTime = Date.now(); + debug('Uglify step 1 took %dms', endTime - startTime); + deferred.resolve(toplevel_ast); + } catch(err) { + debug('JS syntax error, Uglify\'s parser failed (step 1)'); deferred.reject(err); } return deferred.promise; } + function splittedUglifyStep2(toplevel) { + var deferred = Q.defer(); + var startTime = Date.now(); + + toplevel.figure_out_scope(); + + var endTime = Date.now(); + debug('Uglify step 2 took %dms', endTime - startTime); + deferred.resolve(toplevel); + return deferred.promise; + } + + function splittedUglifyStep3(toplevel) { + var deferred = Q.defer(); + var startTime = Date.now(); + + var compressor = UglifyJS.Compressor({warnings: false}); + var compressed_ast = toplevel.transform(compressor); + + var endTime = Date.now(); + debug('Uglify step 3 took %dms', endTime - startTime); + deferred.resolve(compressed_ast); + return deferred.promise; + } + + function splittedUglifyStep4(compressed_ast) { + var deferred = Q.defer(); + var startTime = Date.now(); + + compressed_ast.figure_out_scope(); + + var endTime = Date.now(); + debug('Uglify step 4 took %dms', endTime - startTime); + deferred.resolve(compressed_ast); + return deferred.promise; + } + + function splittedUglifyStep5(compressed_ast) { + var deferred = Q.defer(); + var startTime = Date.now(); + + compressed_ast.compute_char_frequency(); + + var endTime = Date.now(); + debug('Uglify step 5 took %dms', endTime - startTime); + deferred.resolve(compressed_ast); + return deferred.promise; + } + + function splittedUglifyStep6(compressed_ast) { + var deferred = Q.defer(); + var startTime = Date.now(); + + compressed_ast.mangle_names(); + + var endTime = Date.now(); + debug('Uglify step 6 took %dms', endTime - startTime); + deferred.resolve(compressed_ast); + return deferred.promise; + } + + function splittedUglifyStep7(compressed_ast) { + var deferred = Q.defer(); + var startTime = Date.now(); + + var code = compressed_ast.print_to_string(); + + var endTime = Date.now(); + debug('Uglify step 7 took %dms', endTime - startTime); + deferred.resolve(code); + return deferred.promise; + } + // Clear-css function minifyCss(body) { var deferred = Q.defer(); @@ -179,6 +285,44 @@ var FileMinifier = function() { return deferred.promise; } + // Avoid loosing time trying to compress some JS libraries known as already compressed + function isKnownAsMinified(url) { + var result = false; + + // Twitter + result = result || /^https?:\/\/platform\.twitter\.com\/widgets\.js/.test(url); + + // Facebook + result = result || /^https:\/\/connect\.facebook\.net\/[^\/]*\/(sdk|all)\.js/.test(url); + + // Google +1 + result = result || /^https:\/\/apis\.google\.com\/js\/plusone\.js/.test(url); + + // jQuery CDN + result = result || /^https?:\/\/code\.jquery\.com\/.*\.min.js/.test(url); + + + if (result === true) { + debug('This file is known as already minified. Skipping minification: %s', url); + } + + return result; + } + + // Avoid loosing tome trying to compress JS files if they alreay look minified + // by counting the number of lines compared to the total size. + // Less than 1000kb per line is suspicious + function looksAlreadyMinified(code) { + var linesCount = code.split(/\r\n|\r|\n/).length; + var linesRatio = code.length / linesCount / 1024; + var looksMinified = linesRatio > 1; + + debug('Lines ratio is %d KB per line', linesRatio.toFixed(1)); + debug(looksMinified ? 'It looks already minified' : 'It doesn\'t look minified'); + + return looksMinified; + } + function entryTypeCanBeMinified(entry) { return entry.isJS || entry.isCSS || entry.isHTML; } @@ -189,7 +333,8 @@ var FileMinifier = function() { minifyCss: minifyCss, minifyHtml: minifyHtml, gainIsEnough: gainIsEnough, - entryTypeCanBeMinified: entryTypeCanBeMinified + entryTypeCanBeMinified: entryTypeCanBeMinified, + isKnownAsMinified: isKnownAsMinified }; }; diff --git a/lib/tools/weightChecker/gzipCompressor.js b/lib/tools/weightChecker/gzipCompressor.js index 15e0e4d..9981391 100644 --- a/lib/tools/weightChecker/gzipCompressor.js +++ b/lib/tools/weightChecker/gzipCompressor.js @@ -15,7 +15,7 @@ var GzipCompressor = function() { function gzipUncompressedFile(entry) { var deferred = Q.defer(); - if (entryTypeCanBeGzipped(entry) && entry.weightCheck && !entry.weightCheck.isCompressed) { + if (entryTypeCanBeGzipped(entry) && entry.weightCheck && !entry.weightCheck.isCompressed && entry.weightCheck.body) { debug('Compression missing, trying to gzip file %s', entry.url); var uncompressedSize = entry.weightCheck.uncompressedSize; @@ -65,13 +65,8 @@ var GzipCompressor = function() { } else { var compressedSize = buffer.length; - if (gainIsEnough(uncompressedSize, compressedSize)) { - debug('Correctly gziped the minified file, was %d and is now %d bytes', uncompressedSize, compressedSize); - - entry.weightCheck.afterOptimizationAndCompression = compressedSize; - } else { - debug('Gzip gain is not enough on minified file, was %d and is now %d bytes', uncompressedSize, compressedSize); - } + debug('Correctly gziped the minified file, was %d and is now %d bytes', uncompressedSize, compressedSize); + entry.weightCheck.afterOptimizationAndCompression = compressedSize; deferred.resolve(entry); } diff --git a/lib/tools/weightChecker/weightChecker.js b/lib/tools/weightChecker/weightChecker.js index ca3cba5..3611b15 100644 --- a/lib/tools/weightChecker/weightChecker.js +++ b/lib/tools/weightChecker/weightChecker.js @@ -20,7 +20,7 @@ var gzipCompressor = require('./gzipCompressor'); var WeightChecker = function() { var MAX_PARALLEL_DOWNLOADS = 10; - var REQUEST_TIMEOUT = 10000; // 10 seconds + var REQUEST_TIMEOUT = 15000; // 15 seconds // This function will re-download every asset and check if it could be optimized @@ -195,17 +195,24 @@ var WeightChecker = function() { }; requests.forEach(function(req) { - if (req.weightCheck.uncompressedSize && fileMinifier.entryTypeCanBeMinified(req) && req.weightCheck.isOptimized === false) { - var gain = req.weightCheck.uncompressedSize - req.weightCheck.optimized; + if (req.weightCheck.bodySize > 0 && fileMinifier.entryTypeCanBeMinified(req) && req.weightCheck.isOptimized === false) { + var before = req.weightCheck.afterCompression || req.weightCheck.bodySize; + var after = req.weightCheck.afterOptimizationAndCompression || req.weightCheck.afterCompression; + var gain = before - after; - results.totalGain += gain; + if (gain > 200) { + results.totalGain += gain; - results.files.push({ - url: req.url, - original: req.weightCheck.uncompressedSize, - optimized: req.weightCheck.optimized, - gain: gain - }); + results.files.push({ + url: req.url, + original: req.weightCheck.bodySize, + isCompressed: req.weightCheck.isCompressed, + afterCompression: req.weightCheck.afterCompression, + afterOptimizationAndCompression: req.weightCheck.afterOptimizationAndCompression, + optimized: req.weightCheck.optimized, + gain: gain + }); + } } }); diff --git a/test/core/fileMinifierTest.js b/test/core/fileMinifierTest.js index 33e3db3..39d0f9c 100644 --- a/test/core/fileMinifierTest.js +++ b/test/core/fileMinifierTest.js @@ -217,4 +217,23 @@ describe('fileMinifier', function() { }); }); + it('should avoid minifying some JS files known as minified', function() { + fileMinifier.isKnownAsMinified('https://platform.twitter.com/widgets.js').should.equal(true); + fileMinifier.isKnownAsMinified('http://platform.twitter.com/widgets.js').should.equal(true); + + fileMinifier.isKnownAsMinified('https://connect.facebook.net/fr_FR/sdk.js').should.equal(true); + fileMinifier.isKnownAsMinified('https://connect.facebook.net/en_EN/sdk.js').should.equal(true); + fileMinifier.isKnownAsMinified('https://connect.facebook.net/fr_FR/all.js').should.equal(true); + fileMinifier.isKnownAsMinified('https://connect.facebook.net/en_EN/all.js').should.equal(true); + + fileMinifier.isKnownAsMinified('https://apis.google.com/js/plusone.js').should.equal(true); + + fileMinifier.isKnownAsMinified('https://code.jquery.com/jquery-2.1.4.min.js').should.equal(true); + fileMinifier.isKnownAsMinified('http://code.jquery.com/jquery-2.1.4.min.js').should.equal(true); + fileMinifier.isKnownAsMinified('https://code.jquery.com/jquery-2.1.4.js').should.equal(false); + fileMinifier.isKnownAsMinified('http://code.jquery.com/jquery-2.1.4.js').should.equal(false); + + fileMinifier.isKnownAsMinified('http://anydomain.com/anyurl').should.equal(false); + }); + });