Start working on CDN detection

lib/tools/redownload/cdnDetector/cdnDetector.js

@@ -0,0 +1,150 @@
+var debug               = require('debug')('ylt:cdnDetector');
+var Q                   = require('q');
+var url                 = require('url');
+var dns                 = require('dns');
+var cdnDetector         = require('cdn-detector');
+
+// This module checks my multiple ways if a request was served by a CDN
+
+var CDNDetector = function() {
+
+    function detectCDN(entry) {
+        var urlObject = url.parse(entry.url);
+        debug('Start checking CDN for %s, host is %s', entry.url, urlObject.host);
+
+        // 1st, the fastest check: the domain is directly recognized as a CDN
+        return checkDomainDirectly(entry)
+
+        .fail(function() {
+
+            // 2nd, check the response headers
+            return checkHeaders(entry)
+
+            .fail(function() {
+
+                // 3rd, check the Name Servers
+                return checkNameServers(entry)
+
+                .fail(function(entry) {
+
+                    // None of the above worked, we didn't find any trace of a CDN
+                    return entry;
+                });
+            })
+        });
+    }
+
+    return {
+        detectCDN: detectCDN
+    };
+
+    // Compares the URL's domain to a list of know CDN, just in case
+    function checkDomainDirectly(entry) {
+        var deferred = Q.defer();
+
+        var result = checkHostname(url.parse(entry.url).host);
+
+        if (result === null) {
+            deferred.reject(entry);
+        } else {
+            entry.weightCheck.cdn = result;
+            deferred.resolve(entry);
+        }
+            
+        return deferred.promise;
+    }
+
+    // Compares a list of HTTP response headers to a matching list
+    // For example, if there is an X-Akamai-Request-Id header, we know it is Akamai
+    function checkHeaders(entry) {
+        var deferred = Q.defer();
+
+        var result = cdnDetector.detectFromHeaders(entry.weightCheck.headers);
+        debug('Headers of %s were compared to the list. Result: %s', entry.url, JSON.stringify(result));
+
+        if (result === null) {
+            deferred.reject(entry);
+        } else {
+            entry.weightCheck.cdn = result.cdn;
+            deferred.resolve(entry);
+        }
+            
+        return deferred.promise;
+    }
+
+    function checkNameServers(entry) {
+        var deferred = Q.defer();
+        var hostname = url.parse(entry.url).host;
+
+        // Send a request to get the Name Servers list
+        // https://nodejs.org/api/dns.html#dns_dns_resolvens_hostname_callback
+        dns.resolveNs(hostname, function(error, nameServers) {
+            if (error) {
+                debug(error);
+                debug('Could not find Name Servers for %s', hostname);
+                deferred.reject(entry);
+            } else {
+                debug('Name Servers for %s are ["%s"]', hostname, nameServers.join('", "'));
+
+                var cdn = nameServers.map(checkHostname).find(function(entry) {
+                    return entry !== null;
+                });
+
+                if (cdn) {
+                    entry.weightCheck.cdn = cdn;
+                    deferred.resolve(entry);
+                } else {
+                    deferred.reject(entry);
+                }
+            }
+        });
+
+        return deferred.promise;
+    }
+
+    // Compares the hostname to a matching list of hostnames
+    function checkHostname(hostname) {
+        var result = cdnDetector.detectFromHostname(hostname);
+        debug('Hostname %s was compared to the list. Result: %s', hostname, JSON.stringify(result));
+        return result ? result.cdn : null;
+    }
+
+    // Find an IP address with a DNS lookup
+    function getIP(hostname) {
+        var deferred = Q.defer();
+
+        // https://nodejs.org/api/dns.html#dns_dns_lookup_hostname_options_callback
+        dns.lookup(hostname, {family: 4}, function(error, address) {
+            if (error) {
+                debug('Could not find IP address for %s', hostname);
+            } else {
+                debug('The IP address for %s is %s', hostname, address);
+            }
+            deferred.resolve(error ? null : address);
+        });
+
+        return deferred.promise;
+    }
+
+    // Compares the IP address agains a list of IP ranges
+    function checkIPAddress(address) {
+        var deferred = Q.defer();
+
+        var result = cdnDetector.detectFromHeaders(headers);
+        debug('Headers were compared to the list. Result: %s', JSON.stringify(result));
+
+        if (result === null) {
+            deferred.reject();
+        } else {
+            deferred.resolve(result.cdn);
+        }
+
+        return deferred.promise;
+    }
+
+    function saveCDN(cdnName) {
+
+    }
+};
+
+module.exports = new CDNDetector();

lib/tools/redownload/cdnDetector/generateIPList.js

@@ -0,0 +1,71 @@
+var http    = require('https');
+var zlib    = require('zlib');
+var fs      = require('fs');
+
+var CDN_ASN_LIST = {
+    13335: 'Cloudflare',
+    15133: 'Verizon',
+    16625: 'Akamai',
+    20446: 'StackPath',
+    20940: 'Akamai',
+    22822: 'Limelight',
+    54113: 'Fastly'
+};
+
+// Downloads a file and ungzip ip directly
+function getGzipped(url, callback) {
+    var buffer = [];
+    http.get(url, function(res) {
+        var gunzip = zlib.createGunzip();            
+        res.pipe(gunzip);
+        gunzip.on('data', function(data) {
+            buffer.push(data.toString())
+        }).on('end', function() {
+            callback(null, buffer.join('')); 
+        }).on('error', function(e) {
+            callback(e);
+        })
+    }).on('error', function(e) {
+        callback(e)
+    });
+}
+
+// File provided here: https://iptoasn.com/
+console.log('Start downloading the file...');
+getGzipped('https://iptoasn.com/data/ip2asn-v4.tsv.gz', function(err, data) {
+    
+    if (err) {
+        console.log(err);
+        return;
+    }
+
+    console.log('Download complete, let\'s start parsing...');
+    parseFile(data);
+});
+
+// Parse the CSV formated file and grag only the interesting IP ranges
+function parseFile(data) {
+    var results = [];
+
+    var allTextLines = data.split(/\r\n|\n/);
+    for (var i = 0; i < allTextLines.length; i++) {
+        var lineData = allTextLines[i].split('\t');
+        
+        var cdnName = CDN_ASN_LIST[lineData[2]];
+        if (cdnName) {
+
+            // We found an ASN number from our list. Let's save it!
+            results.push({
+                rangeStart: lineData[0],
+                rangeEnd: lineData[1],
+                cdn: cdnName
+            });
+        }
+    }
+
+    console.log('%d IP ranges found', results.length);
+
+    var outputFile = __dirname + '/cdn-ip-list.json';
+    fs.writeFileSync(outputFile, JSON.stringify(results, null, 2));
+    console.log('File saved here: %s', outputFile);
+}

package.json

@@ -29,6 +29,7 @@
     "angular-sanitize": "1.7.7",
     "async": "2.6.1",
     "body-parser": "1.18.3",
+    "cdn-detector": "0.1.6",
     "chart.js": "2.7.3",
     "clean-css": "4.2.1",
     "color-diff": "1.1.0",