YellowLabTools/lib/tools/weightChecker.js

/*
 * Redownloading every files after Phantomas has finished
 * Checks weight and every kind of compression
 *
 */


var debug = require('debug')('ylt:weightChecker');

var request = require('request');
var http    = require('http');
var async   = require('async');
var Q       = require('Q');
var zlib    = require('zlib');

var WeightChecker = function() {

    var MAX_PARALLEL_DOWNLOADS = 10;
    var REQUEST_TIMEOUT = 10000; // 10 seconds

    function recheckAllFiles(data) {
        var deferred = Q.defer();

        var requestsList = JSON.parse(data.toolsResults.phantomas.offenders.requestsList);
        delete data.toolsResults.phantomas.metrics.requestsList;
        delete data.toolsResults.phantomas.offenders.requestsList;

        // Transform every request into a download function with a callback when done
        var redownloadList = requestsList.map(function(entry) {
            return function(callback) {
                redownloadEntry(entry, callback);
            };
        });

        // Lanch all redownload functions and wait for completion
        async.parallelLimit(redownloadList, MAX_PARALLEL_DOWNLOADS, function(err, results) {
            if (err) {
                debug(err);
                deferred.reject(err);
            } else {
                debug('All files checked');
                
                var metrics = {};
                var offenders = {};


                // Total weight
                offenders.totalWeight = listRequestWeight(results);
                metrics.totalWeight = offenders.totalWeight.totalWeight;


                data.toolsResults.weightChecker = {
                    metrics: metrics,
                    offenders: offenders
                };

                deferred.resolve(data);
            }
        });

        return deferred.promise;
    }


    function listRequestWeight(requests) {
        var results = {
            totalWeight: 0,
            byType: {
                html: {
                    totalWeight: 0,
                    requests: []
                },
                css: {
                    totalWeight: 0,
                    requests: []
                },
                js: {
                    totalWeight: 0,
                    requests: []
                },
                json: {
                    totalWeight: 0,
                    requests: []
                },
                image: {
                    totalWeight: 0,
                    requests: []
                },
                video: {
                    totalWeight: 0,
                    requests: []
                },
                webfont: {
                    totalWeight: 0,
                    requests: []
                },
                other: {
                    totalWeight: 0,
                    requests: []
                }
            }
        };

        requests.forEach(function(req) {
            var weight = (typeof req.weightCheck.bodySize === 'number') ? req.weightCheck.bodySize + req.weightCheck.headersSize : req.contentLength;
            var type = req.type || 'other';

            results.totalWeight += weight;
            results.byType[type].totalWeight += weight;

            results.byType[type].requests.push({
                url: req.url,
                weight: weight
            });
        });

        return results;
    }


    function redownloadEntry(entry, callback) {
        
        function onError(message) {
            debug('Could not download %s Error: %s', entry.url, message);
            entry.weightCheck = {
                message: message
            };
            setImmediate(function() {
                callback(null, entry);
            });
        }

        if (entry.method !== 'GET') {
            onError('only downloading GET');
            return;
        }

        if (entry.status !== 200) {
            onError('only downloading requests with status code 200');
            return;
        }


        debug('Downloading %s', entry.url);

        // Always add a gzip header before sending, in case the server listens to it
        var reqHeaders = entry.requestHeaders;
        reqHeaders['Accept-Encoding'] = 'gzip, deflate';

        var requestOptions = {
            method: entry.method,
            url: entry.url,
            headers: reqHeaders,
            timeout: REQUEST_TIMEOUT
        };

        download(requestOptions, function(error, result) {
            if (error) {
                if (error.code === 'ETIMEDOUT') {
                    onError('timeout after ' + REQUEST_TIMEOUT + 'ms');
                } else {
                    onError('error while downloading: ' + error.code);
                }
                return;
            }
                
            debug('%s downloaded correctly', entry.url);

            entry.weightCheck = result;
            callback(null, entry);
        });
    }

    // Inspired by https://github.com/cvan/fastHAR-api/blob/10cec585/app.js
    function download(requestOptions, callback) {

        var statusCode;

        request(requestOptions)

        .on('response', function(res) {
            
            // Raw headers were added in NodeJS v0.12
            // (https://github.com/joyent/node/issues/4844), but let's
            // reconstruct them for backwards compatibility.
            var rawHeaders = ('HTTP/' + res.httpVersion + ' ' + res.statusCode +
                              ' ' + http.STATUS_CODES[res.statusCode] + '\r\n');
            Object.keys(res.headers).forEach(function(headerKey) {
                rawHeaders += headerKey + ': ' + res.headers[headerKey] + '\r\n';
            });
            rawHeaders += '\r\n';

            var uncompressedSize = 0;  // size after uncompression
            var bodySize = 0;  // bytes size over the wire
            var body = '';  // plain text body (after uncompressing gzip/deflate)
            var isCompressed = false;

            function tally() {

                if (statusCode !== 200) {
                    callback({code: statusCode});
                    return;
                }

                var result = {
                    body: body,
                    headersSize: Buffer.byteLength(rawHeaders, 'utf8'),
                    bodySize: bodySize,
                    isCompressed: isCompressed,
                    uncompressedSize: uncompressedSize
                };

                callback(null, result);
            }

            switch (res.headers['content-encoding']) {
                case 'gzip':
                    var gzip = zlib.createGunzip();

                    gzip.on('data', function (data) {
                        body += data;
                        uncompressedSize += data.length;
                    }).on('end', function () {
                        isCompressed = true;
                        tally();
                    });

                    res.on('data', function (data) {
                        bodySize += data.length;
                    }).pipe(gzip);

                    break;
                case 'deflate':
                    var deflate = zlib.createInflate();

                    deflate.on('data', function (data) {
                        body += data;
                        uncompressedSize += data.length;
                    }).on('end', function () {
                        isCompressed = true;
                        tally();
                    });

                    res.on('data', function (data) {
                        bodySize += data.length;
                    }).pipe(deflate);

                    break;
                default:
                    res.on('data', function (data) {
                        body += data;
                        uncompressedSize += data.length;
                        bodySize += data.length;
                    }).on('end', function () {
                        tally();
                    });

                    break;
            }
        })

        .on('response', function(response) {
            statusCode = response.statusCode;
        })

        .on('error', function(err) {
            callback(err);
        });
    }

    return {
        recheckAllFiles: recheckAllFiles,
        listRequestWeight: listRequestWeight,
        redownloadEntry: redownloadEntry,
        download: download
    };
};

module.exports = new WeightChecker();