add netlas module

api-keys.yaml

@@ -30,6 +30,9 @@ apikeys:
   intelx:
     key:
 
+  netlas:
+    key:
+
   pentestTools:
     key:
 

theHarvester/__main__.py

@@ -30,14 +30,13 @@ async def start(rest_args: Optional[argparse.Namespace] = None):
     parser.add_argument('-e', '--dns-server', help='DNS server to use for lookup.')
     parser.add_argument('-t', '--take-over', help='Check for takeovers.', default=False, action='store_true')
     # TODO add dns resolver flag
-    parser.add_argument('-r', '--dns-resolve', help='Perform DNS resolution on subdomains with a resolver list or passed in resolvers, default False.', default="",
-                        type=str, nargs='?')
+    parser.add_argument('-r', '--dns-resolve', help='Perform DNS resolution on subdomains with a resolver list or passed in resolvers, default False.', default="", type=str, nargs='?')
     parser.add_argument('-n', '--dns-lookup', help='Enable DNS server lookup, default False.', default=False, action='store_true')
     parser.add_argument('-c', '--dns-brute', help='Perform a DNS brute force on the domain.', default=False, action='store_true')
     parser.add_argument('-f', '--filename', help='Save the results to an XML and JSON file.', default='', type=str)
     parser.add_argument('-b', '--source', help='''anubis, baidu, bevigil, binaryedge, bing, bingapi, bufferoverun, brave,
                             censys, certspotter, criminalip, crtsh, dnsdumpster, duckduckgo, fullhunt, github-code,
-                            hackertarget, hunter, hunterhow, intelx, otx, pentesttools, projectdiscovery,
+                            hackertarget, hunter, hunterhow, intelx, netlas, otx, pentesttools, projectdiscovery,
                             rapiddns, rocketreach, securityTrails, sitedossier, subdomainfinderc99, threatminer, urlscan,
                             virustotal, yahoo, zoomeye''')
 
@@ -407,6 +406,15 @@ async def start(rest_args: Optional[argparse.Namespace] = None):
                         else:
                             print(f'An exception has occurred in Intelx search: {e}')
 
+                elif engineitem == 'netlas':
+                    from theHarvester.discovery import netlas
+                    try:
+                        netlas_search = netlas.SearchNetlas(word)
+                        stor_lst.append(store(netlas_search, engineitem, store_host=True, store_ip=True))
+                    except Exception as e:
+                        if isinstance(e, MissingKey):
+                            print(e)
+
                 elif engineitem == 'otx':
                     from theHarvester.discovery import otxsearch
                     try:

theHarvester/discovery/bevigil.py

@@ -1,3 +1,4 @@
+from theHarvester.discovery.constants import MissingKey
 from theHarvester.lib.core import *
 from typing import Set
 
@@ -9,6 +10,8 @@ class SearchBeVigil:
         self.totalhosts: Set = set()
         self.interestingurls: Set = set()
         self.key = Core.bevigil_key()
+        if self.key is None:
+            raise MissingKey('bevigil')
         self.proxy = False
 
     async def do_search(self) -> None:

theHarvester/discovery/netlas.py

@@ -0,0 +1,31 @@
+import pprint
+
+from theHarvester.discovery.constants import MissingKey
+from theHarvester.lib.core import *
+from typing import Set
+
+
+class SearchNetlas:
+
+    def __init__(self, word) -> None:
+        self.word = word
+        self.totalhosts: List = []
+        self.totalips: List = []
+        self.key = Core.netlas_key()
+        if self.key is None:
+            raise MissingKey('netlas')
+        self.proxy = False
+
+    async def do_search(self) -> None:
+        api = f'https://app.netlas.io/api/domains/?q=*.{self.word}&source_type=include&start=0&fields=*'
+        headers = {'X-API-Key': self.key}
+        response = await AsyncFetcher.fetch_all([api], json=True, headers=headers, proxy=self.proxy)
+        for domain in response[0]['items']:
+            self.totalhosts.append(domain['data']['domain'])
+
+    async def get_hostnames(self) -> List:
+        return self.totalhosts
+
+    async def process(self, proxy: bool = False) -> None:
+        self.proxy = proxy
+        await self.do_search()

theHarvester/lib/core.py

@@ -72,6 +72,10 @@ class Core:
     def intelx_key() -> str:
         return Core.api_keys()['intelx']['key']
 
+    @staticmethod
+    def netlas_key() -> str:
+        return Core.api_keys()['netlas']['key']
+
     @staticmethod
     def pentest_tools_key() -> str:
         return Core.api_keys()['pentestTools']['key']
@@ -154,6 +158,7 @@ class Core:
                             'hunter',
                             'hunterhow',
                             'intelx',
+                            'netlas',
                             'otx',
                             'pentesttools',
                             'projectdiscovery',