New rule "Identical content" (#181)

lib/tools/redownload/redownload.js

@@ -11,6 +11,7 @@ var http                = require('http');
 var zlib                = require('zlib');
 var async               = require('async');
 var request             = require('request');
+var md5                 = require('md5');
 
 var imageOptimizer      = require('./imageOptimizer');
 var fileMinifier        = require('./fileMinifier');
@@ -106,9 +107,9 @@ var Redownload = function() {
                 metrics.emptyRequests = offenders.emptyRequests.length;
 
 
-                // Now emove unwanted responses (redirections)
+                // Now remove unwanted responses (redirections and empty files)
                 results = results.filter(function(result) {
-                    return (result.status < 300 || result.status >= 400);
+                    return ((result.status < 300 || result.status >= 400) && result.weightCheck.bodySize > 0);
                 });
 
 
@@ -128,6 +129,11 @@ var Redownload = function() {
                 offenders.smallRequests = listSmallRequests(results);
                 metrics.smallRequests = offenders.smallRequests.total;
 
+                // Detect identical files
+                offenders.identicalFiles = listIdenticalFiles(results);
+                metrics.identicalFiles = offenders.identicalFiles.avoidableRequests;
+
+
                 data.toolsResults.redownload = {
                     metrics: metrics,
                     offenders: offenders
@@ -368,6 +374,45 @@ var Redownload = function() {
         return results;
     }
 
+    function listIdenticalFiles(requests) {
+        var hashes = {};
+        var list = [];
+        var avoidableRequestsCount = 0;
+
+        requests.forEach(function(req) {
+            var requestHash = md5(req.weightCheck.body);
+            
+            // Try to exclude tracking pixels
+            if (req.weightCheck.bodySize < 80 && req.type === 'image') {
+                return;
+            }
+
+            if (!hashes[requestHash]) {
+                hashes[requestHash] = {
+                    weight: req.weightCheck.bodySize,
+                    urls: []
+                };
+            }
+            
+            if (hashes[requestHash].urls.indexOf(req.url) === -1) {
+                hashes[requestHash].urls.push(req.url);
+            }
+        });
+
+        for (var hash in hashes) {
+            if (hashes[hash].urls.length > 1) {
+                list.push(hashes[hash]);
+                avoidableRequestsCount += hashes[hash].urls.length - 1;
+            }
+        }
+
+        return {
+            avoidableRequests: avoidableRequestsCount,
+            count: list.length,
+            list: list
+        };
+    }
+
 
     function redownloadEntry(entry, httpAuth) {
         var deferred = Q.defer();