WEBDIENSTE/YellowLabTools
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Optimizing JS minification step that takes too long

Browse Source
This commit is contained in:
Gaël Métais
2015-06-19 14:05:21 +02:00
parent d4815e855e
commit 583b6b9a55
4 changed files with 191 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/tools/weightChecker/fileMinifier.js
+152 -7
View File
@@ -20,12 +20,12 @@ var FileMinifier = function() {
var fileSize = entry.weightCheck.uncompressedSize;
debug('Let\'s try to optimize %s', entry.url);
debug('Current file size is %d', fileSize);
var startTime = Date.now();
if (entry.isJS) {
if (entry.isJS && !isKnownAsMinified(entry.url) && !looksAlreadyMinified(entry.weightCheck.body)) {
debug('File is a JS');
// Starting softly with a lossless compression
return minifyJs(entry.weightCheck.body)
.then(function(newFile) {
@@ -34,6 +34,8 @@ var FileMinifier = function() {
return entry;
}
var endTime = Date.now();
var newFileSize = newFile.length;
debug('JS minification complete for %s', entry.url);
@@ -56,7 +58,6 @@ var FileMinifier = function() {
debug('File is a CSS');
// Starting softly with a lossless compression
return minifyCss(entry.weightCheck.body)
.then(function(newFile) {
@@ -65,6 +66,9 @@ var FileMinifier = function() {
return entry;
}
var endTime = Date.now();
debug('CSS minification took %dms', endTime - startTime);
var newFileSize = newFile.length;
debug('CSS minification complete for %s', entry.url);
@@ -87,7 +91,6 @@ var FileMinifier = function() {
debug('File is an HTML');
// Starting softly with a lossless compression
return minifyHtml(entry.weightCheck.body)
.then(function(newFile) {
@@ -96,6 +99,9 @@ var FileMinifier = function() {
return entry;
}
var endTime = Date.now();
debug('HTML minification took %dms', endTime - startTime);
var newFileSize = newFile.length;
debug('HTML minification complete for %s', entry.url);
@@ -132,18 +138,118 @@ var FileMinifier = function() {
// Uglify
function minifyJs(body) {
// Splitting the Uglify function because it sometime takes too long (more than 10 seconds)
// I hope that, by splitting, it can be a little more asynchronous, so the application doesn't freeze.
return splittedUglifyStep1(body)
.delay(1)
.then(splittedUglifyStep2)
.delay(1)
.then(splittedUglifyStep3)
.delay(1)
.then(splittedUglifyStep4)
.delay(1)
.then(splittedUglifyStep5)
.delay(1)
.then(splittedUglifyStep6)
.delay(1)
.then(splittedUglifyStep7);
}
function splittedUglifyStep1(code) {
var deferred = Q.defer();
var startTime = Date.now();
try {
var result = UglifyJS.minify(body, {fromString: true});
deferred.resolve(result.code);
var toplevel_ast = UglifyJS.parse(code);
var endTime = Date.now();
debug('Uglify step 1 took %dms', endTime - startTime);
deferred.resolve(toplevel_ast);
} catch(err) {
debug('JS syntax error, Uglify\'s parser failed (step 1)');
deferred.reject(err);
}
return deferred.promise;
}
function splittedUglifyStep2(toplevel) {
var deferred = Q.defer();
var startTime = Date.now();
toplevel.figure_out_scope();
var endTime = Date.now();
debug('Uglify step 2 took %dms', endTime - startTime);
deferred.resolve(toplevel);
return deferred.promise;
}
function splittedUglifyStep3(toplevel) {
var deferred = Q.defer();
var startTime = Date.now();
var compressor = UglifyJS.Compressor({warnings: false});
var compressed_ast = toplevel.transform(compressor);
var endTime = Date.now();
debug('Uglify step 3 took %dms', endTime - startTime);
deferred.resolve(compressed_ast);
return deferred.promise;
}
function splittedUglifyStep4(compressed_ast) {
var deferred = Q.defer();
var startTime = Date.now();
compressed_ast.figure_out_scope();
var endTime = Date.now();
debug('Uglify step 4 took %dms', endTime - startTime);
deferred.resolve(compressed_ast);
return deferred.promise;
}
function splittedUglifyStep5(compressed_ast) {
var deferred = Q.defer();
var startTime = Date.now();
compressed_ast.compute_char_frequency();
var endTime = Date.now();
debug('Uglify step 5 took %dms', endTime - startTime);
deferred.resolve(compressed_ast);
return deferred.promise;
}
function splittedUglifyStep6(compressed_ast) {
var deferred = Q.defer();
var startTime = Date.now();
compressed_ast.mangle_names();
var endTime = Date.now();
debug('Uglify step 6 took %dms', endTime - startTime);
deferred.resolve(compressed_ast);
return deferred.promise;
}
function splittedUglifyStep7(compressed_ast) {
var deferred = Q.defer();
var startTime = Date.now();
var code = compressed_ast.print_to_string();
var endTime = Date.now();
debug('Uglify step 7 took %dms', endTime - startTime);
deferred.resolve(code);
return deferred.promise;
}
// Clear-css
function minifyCss(body) {
var deferred = Q.defer();
@@ -179,6 +285,44 @@ var FileMinifier = function() {
return deferred.promise;
}
// Avoid loosing time trying to compress some JS libraries known as already compressed
function isKnownAsMinified(url) {
var result = false;
// Twitter
result = result || /^https?:\/\/platform\.twitter\.com\/widgets\.js/.test(url);
// Facebook
result = result || /^https:\/\/connect\.facebook\.net\/[^\/]*\/(sdk|all)\.js/.test(url);
// Google +1
result = result || /^https:\/\/apis\.google\.com\/js\/plusone\.js/.test(url);
// jQuery CDN
result = result || /^https?:\/\/code\.jquery\.com\/.*\.min.js/.test(url);
if (result === true) {
debug('This file is known as already minified. Skipping minification: %s', url);
}
return result;
}
// Avoid loosing tome trying to compress JS files if they alreay look minified
// by counting the number of lines compared to the total size.
// Less than 1000kb per line is suspicious
function looksAlreadyMinified(code) {
var linesCount = code.split(/\r\n|\r|\n/).length;
var linesRatio = code.length / linesCount / 1024;
var looksMinified = linesRatio > 1;
debug('Lines ratio is %d KB per line', linesRatio.toFixed(1));
debug(looksMinified ? 'It looks already minified' : 'It doesn\'t look minified');
return looksMinified;
}
function entryTypeCanBeMinified(entry) {
return entry.isJS || entry.isCSS || entry.isHTML;
}
@@ -189,7 +333,8 @@ var FileMinifier = function() {
minifyCss: minifyCss,
minifyHtml: minifyHtml,
gainIsEnough: gainIsEnough,
entryTypeCanBeMinified: entryTypeCanBeMinified
entryTypeCanBeMinified: entryTypeCanBeMinified,
isKnownAsMinified: isKnownAsMinified
};
};
lib/tools/weightChecker/gzipCompressor.js
+3 -8
View File
@@ -15,7 +15,7 @@ var GzipCompressor = function() {
function gzipUncompressedFile(entry) {
var deferred = Q.defer();
if (entryTypeCanBeGzipped(entry) && entry.weightCheck && !entry.weightCheck.isCompressed) {
if (entryTypeCanBeGzipped(entry) && entry.weightCheck && !entry.weightCheck.isCompressed && entry.weightCheck.body) {
debug('Compression missing, trying to gzip file %s', entry.url);
var uncompressedSize = entry.weightCheck.uncompressedSize;
@@ -65,13 +65,8 @@ var GzipCompressor = function() {
} else {
var compressedSize = buffer.length;
if (gainIsEnough(uncompressedSize, compressedSize)) {
debug('Correctly gziped the minified file, was %d and is now %d bytes', uncompressedSize, compressedSize);
entry.weightCheck.afterOptimizationAndCompression = compressedSize;
} else {
debug('Gzip gain is not enough on minified file, was %d and is now %d bytes', uncompressedSize, compressedSize);
}
debug('Correctly gziped the minified file, was %d and is now %d bytes', uncompressedSize, compressedSize);
entry.weightCheck.afterOptimizationAndCompression = compressedSize;
deferred.resolve(entry);
}
lib/tools/weightChecker/weightChecker.js
+17 -10
View File
@@ -20,7 +20,7 @@ var gzipCompressor = require('./gzipCompressor');
var WeightChecker = function() {
var MAX_PARALLEL_DOWNLOADS = 10;
var REQUEST_TIMEOUT = 10000; // 10 seconds
var REQUEST_TIMEOUT = 15000; // 15 seconds
// This function will re-download every asset and check if it could be optimized
@@ -195,17 +195,24 @@ var WeightChecker = function() {
};
requests.forEach(function(req) {
if (req.weightCheck.uncompressedSize && fileMinifier.entryTypeCanBeMinified(req) && req.weightCheck.isOptimized === false) {
var gain = req.weightCheck.uncompressedSize - req.weightCheck.optimized;
if (req.weightCheck.bodySize > 0 && fileMinifier.entryTypeCanBeMinified(req) && req.weightCheck.isOptimized === false) {
var before = req.weightCheck.afterCompression || req.weightCheck.bodySize;
var after = req.weightCheck.afterOptimizationAndCompression || req.weightCheck.afterCompression;
var gain = before - after;
results.totalGain += gain;
if (gain > 200) {
results.totalGain += gain;
results.files.push({
url: req.url,
original: req.weightCheck.uncompressedSize,
optimized: req.weightCheck.optimized,
gain: gain
});
results.files.push({
url: req.url,
original: req.weightCheck.bodySize,
isCompressed: req.weightCheck.isCompressed,
afterCompression: req.weightCheck.afterCompression,
afterOptimizationAndCompression: req.weightCheck.afterOptimizationAndCompression,
optimized: req.weightCheck.optimized,
gain: gain
});
}
}
});
test/core/fileMinifierTest.js
+19
View File
@@ -217,4 +217,23 @@ describe('fileMinifier', function() {
});
});
it('should avoid minifying some JS files known as minified', function() {
fileMinifier.isKnownAsMinified('https://platform.twitter.com/widgets.js').should.equal(true);
fileMinifier.isKnownAsMinified('http://platform.twitter.com/widgets.js').should.equal(true);
fileMinifier.isKnownAsMinified('https://connect.facebook.net/fr_FR/sdk.js').should.equal(true);
fileMinifier.isKnownAsMinified('https://connect.facebook.net/en_EN/sdk.js').should.equal(true);
fileMinifier.isKnownAsMinified('https://connect.facebook.net/fr_FR/all.js').should.equal(true);
fileMinifier.isKnownAsMinified('https://connect.facebook.net/en_EN/all.js').should.equal(true);
fileMinifier.isKnownAsMinified('https://apis.google.com/js/plusone.js').should.equal(true);
fileMinifier.isKnownAsMinified('https://code.jquery.com/jquery-2.1.4.min.js').should.equal(true);
fileMinifier.isKnownAsMinified('http://code.jquery.com/jquery-2.1.4.min.js').should.equal(true);
fileMinifier.isKnownAsMinified('https://code.jquery.com/jquery-2.1.4.js').should.equal(false);
fileMinifier.isKnownAsMinified('http://code.jquery.com/jquery-2.1.4.js').should.equal(false);
fileMinifier.isKnownAsMinified('http://anydomain.com/anyurl').should.equal(false);
});
});