diff --git a/discovery/crtsh.py b/discovery/crtsh.py index 8b9a8e35..59d7fa68 100644 --- a/discovery/crtsh.py +++ b/discovery/crtsh.py @@ -1,5 +1,7 @@ import requests import myparser +import time +import random class search_crtsh: @@ -7,28 +9,69 @@ class search_crtsh: self.word = word.replace(' ', '%20') self.results = "" self.totalresults = "" - self.server = "www.google.com" - self.hostname = "www.google.com" - self.userAgent = "(Mozilla/5.0 (Windows; U; Windows NT 6.0;en-US; rv:1.9.2) Gecko/20100116 Firefox/3.7" + self.server = "https://crt.sh/?q=" + self.userAgent = ["(Mozilla/5.0 (Windows; U; Windows NT 6.0;en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36" + , ("Mozilla/5.0 (Linux; Android 7.0; SM-G892A Build/NRD90M; wv) " + + "AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/60.0.3112.107 Mobile Safari/537.36"), + ("Mozilla/5.0 (Windows Phone 10.0; Android 6.0.1; Microsoft; RM-1152) " + + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Mobile Safari/537.36 Edge/15.15254"), + "Mozilla/5.0 (SMART-TV; X11; Linux armv7l) AppleWebKit/537.42 (KHTML, like Gecko) Chromium/25.0.1349.2 Chrome/25.0.1349.2 Safari/537.42", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36 OPR/43.0.2442.991", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36 OPR/48.0.2685.52", + "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko", + "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko", + "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)"] self.quantity = "100" self.counter = 0 + def do_search(self): try: - urly = "https://crt.sh/?q=%25" + self.word + urly = self.server + self.word except Exception as e: print(e) - headers = {'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:34.0) Gecko/20100101 Firefox/34.0'} try: - r=requests.get(urly,headers=headers) + params = {'User-Agent': random.choice(self.userAgent)} + r=requests.get(urly,headers=params) except Exception as e: print(e) - self.results = r.text - self.totalresults += self.results + links = self.get_info(r.text) + for link in links: + params = {'User-Agent': random.choice(self.userAgent)} + r = requests.get(link, headers=params) + time.sleep(1) + self.results = r.text + self.totalresults += self.results + + """ + Function goes through text from base request and parses it for links + @param text requests text + @return list of links + """ + def get_info(self,text): + lines = [] + for line in str(text).splitlines(): + line = line.strip() + if 'id=' in line: + lines.append(line) + links = [] + for i in range(len(lines)): + if i % 2 == 0: #way html is formatted only care about every other one + current = lines[i] + current = current[43:] #43 is not an arbitrary number, the id number always starts at 43rd index + link = '' + for ch in current: + if ch == '"': + break + else: + link += ch + links.append(('https://crt.sh?id=' + str(link))) + return links def get_hostnames(self): - rawres = myparser.parser(self.results, self.word) + rawres = myparser.parser(self.totalresults, self.word) return rawres.hostnames() def process(self): diff --git a/myparser.py b/myparser.py index 40f58692..2d400931 100644 --- a/myparser.py +++ b/myparser.py @@ -1,4 +1,3 @@ -import string import re diff --git a/theHarvester.py b/theHarvester.py index 585054df..b6947473 100644 --- a/theHarvester.py +++ b/theHarvester.py @@ -356,13 +356,13 @@ def start(argv): db.store_all(word,all_hosts,'host','threatcrowd') except Exception: pass - """print("[-] Searching in CRTSH server..") + print("[-] Searching in CRTSH server..") search = crtsh.search_crtsh(word) search.process() hosts = search.get_hostnames() all_hosts.extend(hosts) db=stash.stash_manager() - db.store_all(word,all_hosts,'host','CRTsh')""" + db.store_all(word,all_hosts,'host','CRTsh') print("[-] Searching in Virustotal server..") search = virustotal.search_virustotal(word) @@ -416,7 +416,7 @@ def start(argv): # pass #Results############################################################ - print("\n\033[1;32;40m Harvesting results") + print("\n\033[1;32;40mHarvesting results") print("\n\n[+] Emails found:") print("------------------")