Start working on CDN detection

lib/metadata/policies.js

@@ -875,7 +875,7 @@ var policies = {
             return offenders;
         }
     },
-    "unusedUnicodeRanges": {
+    /*"unusedUnicodeRanges": {
         "tool": "redownload",
         "label": "Unused Unicode ranges",
         "message": "<p>This metric counts the number of unused Unicode ranges inside each font. For example, one font could include Cyrillic glyphs but none of them are used on the page.</p><p>It also reveals the number of ligatures (letters that are represented differently when close to each other) and hidden chars (glyphs not linked to the unicode system that can't be displayed on the web).</p><p>Because of technical limitations, Yellow Lab Tools checks each font against the glyphs of the entire page. As a result, estimated use is >= to reality. For example, if you read that 10 glyphs are \"possibly used\", it means that these 10 glyphs are used on the page but nothing guaranties that they are displayed using this font.</p><p>Tools such as <a href=\"https://www.fontsquirrel.com/tools/webfont-generator\" target=\"_blank\">Font Squirrel</a> can remove some unicode ranges from a font.</p><p>In the case of an icon font, make sure you only keep the icons that are used on the website and to remove the others. Several tools are able to extract SVG images from a font, then some other tools can generate a font from the SVGs you want to keep.</p>",
@@ -886,7 +886,7 @@ var policies = {
         "offendersTransformFn": function(offenders) {
             return offenders;
         }
-    },
+    },*/
     "nonWoff2Fonts": {
         "tool": "redownload",
         "label": "WOFF 2",

lib/metadata/scoreProfileGeneric.json

@@ -86,7 +86,6 @@
             "policies": {
                 "fontsCount": 1,
                 "heavyFonts": 0.5,
-                "unusedUnicodeRanges": 0.5,
                 "nonWoff2Fonts": 0.5
             }
         },

lib/runner.js

@@ -78,12 +78,6 @@ var Runner = function(params) {
             milestone: 'redownload'
         });
 
-        // Fix: don't display Unicode ranges if the module is not present in Phantomas
-        if (!data.toolsResults.phantomas.metrics.charactersCount) {
-            delete data.toolsResults.redownload.metrics.unusedUnicodeRanges;
-            delete data.toolsResults.redownload.offenders.unusedUnicodeRanges;
-        }
-
         // Rules checker
         var policies = require('./metadata/policies');
         data.rules = rulesChecker.check(data, policies);

lib/tools/redownload/cdnDetector/cdnDetector.js

@@ -0,0 +1,150 @@
+var debug               = require('debug')('ylt:cdnDetector');
+var Q                   = require('q');
+var url                 = require('url');
+var dns                 = require('dns');
+var cdnDetector         = require('cdn-detector');
+
+// This module checks my multiple ways if a request was served by a CDN
+
+var CDNDetector = function() {
+
+    function detectCDN(entry) {
+        var urlObject = url.parse(entry.url);
+        debug('Start checking CDN for %s, host is %s', entry.url, urlObject.host);
+
+        // 1st, the fastest check: the domain is directly recognized as a CDN
+        return checkDomainDirectly(entry)
+
+        .fail(function() {
+
+            // 2nd, check the response headers
+            return checkHeaders(entry)
+
+            .fail(function() {
+
+                // 3rd, check the Name Servers
+                return checkNameServers(entry)
+
+                .fail(function(entry) {
+
+                    // None of the above worked, we didn't find any trace of a CDN
+                    return entry;
+                });
+            })
+        });
+    }
+
+    return {
+        detectCDN: detectCDN
+    };
+
+    // Compares the URL's domain to a list of know CDN, just in case
+    function checkDomainDirectly(entry) {
+        var deferred = Q.defer();
+
+        var result = checkHostname(url.parse(entry.url).host);
+
+        if (result === null) {
+            deferred.reject(entry);
+        } else {
+            entry.weightCheck.cdn = result;
+            deferred.resolve(entry);
+        }
+            
+        return deferred.promise;
+    }
+
+    // Compares a list of HTTP response headers to a matching list
+    // For example, if there is an X-Akamai-Request-Id header, we know it is Akamai
+    function checkHeaders(entry) {
+        var deferred = Q.defer();
+
+        var result = cdnDetector.detectFromHeaders(entry.weightCheck.headers);
+        debug('Headers of %s were compared to the list. Result: %s', entry.url, JSON.stringify(result));
+
+        if (result === null) {
+            deferred.reject(entry);
+        } else {
+            entry.weightCheck.cdn = result.cdn;
+            deferred.resolve(entry);
+        }
+            
+        return deferred.promise;
+    }
+
+    function checkNameServers(entry) {
+        var deferred = Q.defer();
+        var hostname = url.parse(entry.url).host;
+
+        // Send a request to get the Name Servers list
+        // https://nodejs.org/api/dns.html#dns_dns_resolvens_hostname_callback
+        dns.resolveNs(hostname, function(error, nameServers) {
+            if (error) {
+                debug(error);
+                debug('Could not find Name Servers for %s', hostname);
+                deferred.reject(entry);
+            } else {
+                debug('Name Servers for %s are ["%s"]', hostname, nameServers.join('", "'));
+
+                var cdn = nameServers.map(checkHostname).find(function(entry) {
+                    return entry !== null;
+                });
+
+                if (cdn) {
+                    entry.weightCheck.cdn = cdn;
+                    deferred.resolve(entry);
+                } else {
+                    deferred.reject(entry);
+                }
+            }
+        });
+
+        return deferred.promise;
+    }
+
+    // Compares the hostname to a matching list of hostnames
+    function checkHostname(hostname) {
+        var result = cdnDetector.detectFromHostname(hostname);
+        debug('Hostname %s was compared to the list. Result: %s', hostname, JSON.stringify(result));
+        return result ? result.cdn : null;
+    }
+
+    // Find an IP address with a DNS lookup
+    function getIP(hostname) {
+        var deferred = Q.defer();
+
+        // https://nodejs.org/api/dns.html#dns_dns_lookup_hostname_options_callback
+        dns.lookup(hostname, {family: 4}, function(error, address) {
+            if (error) {
+                debug('Could not find IP address for %s', hostname);
+            } else {
+                debug('The IP address for %s is %s', hostname, address);
+            }
+            deferred.resolve(error ? null : address);
+        });
+
+        return deferred.promise;
+    }
+
+    // Compares the IP address agains a list of IP ranges
+    function checkIPAddress(address) {
+        var deferred = Q.defer();
+
+        var result = cdnDetector.detectFromHeaders(headers);
+        debug('Headers were compared to the list. Result: %s', JSON.stringify(result));
+
+        if (result === null) {
+            deferred.reject();
+        } else {
+            deferred.resolve(result.cdn);
+        }
+
+        return deferred.promise;
+    }
+
+    function saveCDN(cdnName) {
+
+    }
+};
+
+module.exports = new CDNDetector();

lib/tools/redownload/cdnDetector/generateIPList.js

@@ -0,0 +1,71 @@
+var http    = require('https');
+var zlib    = require('zlib');
+var fs      = require('fs');
+
+var CDN_ASN_LIST = {
+    13335: 'Cloudflare',
+    15133: 'Verizon',
+    16625: 'Akamai',
+    20446: 'StackPath',
+    20940: 'Akamai',
+    22822: 'Limelight',
+    54113: 'Fastly'
+};
+
+// Downloads a file and ungzip ip directly
+function getGzipped(url, callback) {
+    var buffer = [];
+    http.get(url, function(res) {
+        var gunzip = zlib.createGunzip();            
+        res.pipe(gunzip);
+        gunzip.on('data', function(data) {
+            buffer.push(data.toString())
+        }).on('end', function() {
+            callback(null, buffer.join('')); 
+        }).on('error', function(e) {
+            callback(e);
+        })
+    }).on('error', function(e) {
+        callback(e)
+    });
+}
+
+// File provided here: https://iptoasn.com/
+console.log('Start downloading the file...');
+getGzipped('https://iptoasn.com/data/ip2asn-v4.tsv.gz', function(err, data) {
+    
+    if (err) {
+        console.log(err);
+        return;
+    }
+
+    console.log('Download complete, let\'s start parsing...');
+    parseFile(data);
+});
+
+// Parse the CSV formated file and grag only the interesting IP ranges
+function parseFile(data) {
+    var results = [];
+
+    var allTextLines = data.split(/\r\n|\n/);
+    for (var i = 0; i < allTextLines.length; i++) {
+        var lineData = allTextLines[i].split('\t');
+        
+        var cdnName = CDN_ASN_LIST[lineData[2]];
+        if (cdnName) {
+
+            // We found an ASN number from our list. Let's save it!
+            results.push({
+                rangeStart: lineData[0],
+                rangeEnd: lineData[1],
+                cdn: cdnName
+            });
+        }
+    }
+
+    console.log('%d IP ranges found', results.length);
+
+    var outputFile = __dirname + '/cdn-ip-list.json';
+    fs.writeFileSync(outputFile, JSON.stringify(results, null, 2));
+    console.log('File saved here: %s', outputFile);
+}

lib/tools/redownload/redownload.js

@@ -65,8 +65,8 @@ var Redownload = function() {
 
         // Prevent a bug with the font analyzer on empty pages
         var differentCharacters = '';
-        if (data.toolsResults.phantomas.offenders.charactersCount && data.toolsResults.phantomas.offenders.charactersCount.length > 0) {
+        if (data.toolsResults.phantomas.offenders.differentCharacters && data.toolsResults.phantomas.offenders.differentCharacters.length > 0) {
-            differentCharacters = data.toolsResults.phantomas.offenders.charactersCount[0];
+            differentCharacters = data.toolsResults.phantomas.offenders.differentCharacters[0];
         }
 
         // Transform every request into a download function with a callback when done
@@ -900,7 +900,6 @@ var Redownload = function() {
                     var result = {
                         bodyBuffer: body,
                         headersSize: Buffer.byteLength(rawHeaders, 'utf8'),
-                        headers: res.headers,
                         bodySize: bodySize,
                         isCompressed: isCompressed,
                         compressionTool: compressionTool,

package.json

@@ -29,6 +29,7 @@
     "angular-sanitize": "1.7.7",
     "async": "2.6.1",
     "body-parser": "1.18.3",
+    "cdn-detector": "0.1.6",
     "chart.js": "2.7.3",
     "clean-css": "4.2.1",
     "color-diff": "1.1.0",