From 20f22b57bdbd3e924cc022ab7a6f738e3d762e3e Mon Sep 17 00:00:00 2001 From: NotoriousRebel Date: Mon, 26 Nov 2018 14:07:20 -0500 Subject: [PATCH 1/6] fixed local variable error being referenced before defined in googleCSE search, also removed extra whitespace. --- theHarvester.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/theHarvester.py b/theHarvester.py index e11b69f1..479e3862 100644 --- a/theHarvester.py +++ b/theHarvester.py @@ -188,8 +188,8 @@ def start(argv): search.store_results() all_emails = search.get_emails() db=stash.stash_manager() - db.store_all(word,all_hosts,'email','googleCSE') all_hosts = search.get_hostnames() + db.store_all(word,all_hosts,'email','googleCSE') db=stash.stash_manager() db.store_all(word,all_hosts,'host','googleCSE') @@ -389,7 +389,7 @@ def start(argv): # pass #Results############################################################ - print("\n\033[1;32;40m Harvesting results") + print("\n\033[1;32;40mHarvesting results") print("\n\n[+] Emails found:") print("------------------") From f3788c6b4176068c8fa32303cdfa90f8de9fd28d Mon Sep 17 00:00:00 2001 From: NotoriousRebel Date: Tue, 27 Nov 2018 22:05:51 -0500 Subject: [PATCH 2/6] Added method to get crt.sh ids and crawl them. --- discovery/crtsh.py | 33 ++++++++++++++++++++++++++++----- myparser.py | 1 - theHarvester.py | 4 ++-- 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/discovery/crtsh.py b/discovery/crtsh.py index 8b9a8e35..5b4cdab9 100644 --- a/discovery/crtsh.py +++ b/discovery/crtsh.py @@ -7,8 +7,7 @@ class search_crtsh: self.word = word.replace(' ', '%20') self.results = "" self.totalresults = "" - self.server = "www.google.com" - self.hostname = "www.google.com" + self.server = "https://crt.sh/?q=" self.userAgent = "(Mozilla/5.0 (Windows; U; Windows NT 6.0;en-US; rv:1.9.2) Gecko/20100116 Firefox/3.7" self.quantity = "100" self.counter = 0 @@ -16,7 +15,7 @@ class search_crtsh: def do_search(self): try: - urly = "https://crt.sh/?q=%25" + self.word + urly = self.server + self.word except Exception as e: print(e) headers = {'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:34.0) Gecko/20100101 Firefox/34.0'} @@ -24,8 +23,32 @@ class search_crtsh: r=requests.get(urly,headers=headers) except Exception as e: print(e) - self.results = r.text - self.totalresults += self.results + links = self.get_info(r.text) + for link in links: + r = requests.get(link, headers=headers) + self.results = r.text + self.totalresults += self.results + + def get_info(self,text): + lines = [] + for line in str(text).splitlines(): + line = line.strip() + if 'id=' in line: + lines.append(line) + links = [] + for i in range(len(lines)): + if i % 2 == 0: + current = lines[i] + current = current[43:] + link = '' + for ch in current: + if ch == '"': + break + else: + link += ch + links.append(('https://crt.sh?id=' + str(link))) + return links + def get_hostnames(self): rawres = myparser.parser(self.results, self.word) diff --git a/myparser.py b/myparser.py index 40f58692..2d400931 100644 --- a/myparser.py +++ b/myparser.py @@ -1,4 +1,3 @@ -import string import re diff --git a/theHarvester.py b/theHarvester.py index 479e3862..bbb6bd07 100644 --- a/theHarvester.py +++ b/theHarvester.py @@ -337,13 +337,13 @@ def start(argv): db.store_all(word,all_hosts,'host','threatcrowd') except Exception: pass - """print("[-] Searching in CRTSH server..") + print("[-] Searching in CRTSH server..") search = crtsh.search_crtsh(word) search.process() hosts = search.get_hostnames() all_hosts.extend(hosts) db=stash.stash_manager() - db.store_all(word,all_hosts,'host','CRTsh')""" + db.store_all(word,all_hosts,'host','CRTsh') print("[-] Searching in Virustotal server..") search = virustotal.search_virustotal(word) From d8a8cb3ccc3dbb01b0086d34b7a2fbda0aa512eb Mon Sep 17 00:00:00 2001 From: NotoriousRebel Date: Tue, 27 Nov 2018 23:40:30 -0500 Subject: [PATCH 3/6] Confirmed request is returning correct thing and am close to figuring out parser issue. --- discovery/crtsh.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/discovery/crtsh.py b/discovery/crtsh.py index 5b4cdab9..60c162c2 100644 --- a/discovery/crtsh.py +++ b/discovery/crtsh.py @@ -26,8 +26,10 @@ class search_crtsh: links = self.get_info(r.text) for link in links: r = requests.get(link, headers=headers) - self.results = r.text - self.totalresults += self.results + for l in str(r.text).splitlines(): + if self.word in l: + print('concatting l to totalresults: ', l.replace(' ','').replace(';','')) + self.totalresults += l def get_info(self,text): lines = [] From 00c884af135cf0d551689fce1ce3e631c1a685ec Mon Sep 17 00:00:00 2001 From: NotoriousRebel Date: Tue, 27 Nov 2018 23:45:19 -0500 Subject: [PATCH 4/6] Got crtsh search working as problem was in get_hostnames first line replaced self.results to self.totalresults. --- discovery/crtsh.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/discovery/crtsh.py b/discovery/crtsh.py index 60c162c2..622ea3ee 100644 --- a/discovery/crtsh.py +++ b/discovery/crtsh.py @@ -26,10 +26,8 @@ class search_crtsh: links = self.get_info(r.text) for link in links: r = requests.get(link, headers=headers) - for l in str(r.text).splitlines(): - if self.word in l: - print('concatting l to totalresults: ', l.replace(' ','').replace(';','')) - self.totalresults += l + self.results = r.text + self.totalresults += self.results def get_info(self,text): lines = [] @@ -53,7 +51,7 @@ class search_crtsh: def get_hostnames(self): - rawres = myparser.parser(self.results, self.word) + rawres = myparser.parser(self.totalresults, self.word) return rawres.hostnames() def process(self): From 31a77cc199fba1579e3fa0bb23b4dd0cc8a5a1d2 Mon Sep 17 00:00:00 2001 From: NotoriousRebel Date: Wed, 28 Nov 2018 12:45:03 -0500 Subject: [PATCH 5/6] Added docstring to crtsh.py --- discovery/crtsh.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/discovery/crtsh.py b/discovery/crtsh.py index 622ea3ee..4bd3f240 100644 --- a/discovery/crtsh.py +++ b/discovery/crtsh.py @@ -29,6 +29,11 @@ class search_crtsh: self.results = r.text self.totalresults += self.results + """ + Function goes through text from base request and parses it for links + @param text requests text + @return list of links + """ def get_info(self,text): lines = [] for line in str(text).splitlines(): @@ -37,9 +42,9 @@ class search_crtsh: lines.append(line) links = [] for i in range(len(lines)): - if i % 2 == 0: + if i % 2 == 0: #way html is formatted only care about every other one current = lines[i] - current = current[43:] + current = current[43:] #43 is not an arbitrary number, the id number always starts at 43rd index link = '' for ch in current: if ch == '"': @@ -49,7 +54,6 @@ class search_crtsh: links.append(('https://crt.sh?id=' + str(link))) return links - def get_hostnames(self): rawres = myparser.parser(self.totalresults, self.word) return rawres.hostnames() From fcbca16ab108ec36f8050059f630aa2f9d995ce4 Mon Sep 17 00:00:00 2001 From: NotoriousRebel Date: Wed, 28 Nov 2018 14:40:58 -0500 Subject: [PATCH 6/6] Added random selection of user-agent, and sleep inbetween requests for crtsh.py --- discovery/crtsh.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/discovery/crtsh.py b/discovery/crtsh.py index 4bd3f240..59d7fa68 100644 --- a/discovery/crtsh.py +++ b/discovery/crtsh.py @@ -1,5 +1,7 @@ import requests import myparser +import time +import random class search_crtsh: @@ -8,9 +10,21 @@ class search_crtsh: self.results = "" self.totalresults = "" self.server = "https://crt.sh/?q=" - self.userAgent = "(Mozilla/5.0 (Windows; U; Windows NT 6.0;en-US; rv:1.9.2) Gecko/20100116 Firefox/3.7" + self.userAgent = ["(Mozilla/5.0 (Windows; U; Windows NT 6.0;en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36" + , ("Mozilla/5.0 (Linux; Android 7.0; SM-G892A Build/NRD90M; wv) " + + "AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/60.0.3112.107 Mobile Safari/537.36"), + ("Mozilla/5.0 (Windows Phone 10.0; Android 6.0.1; Microsoft; RM-1152) " + + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Mobile Safari/537.36 Edge/15.15254"), + "Mozilla/5.0 (SMART-TV; X11; Linux armv7l) AppleWebKit/537.42 (KHTML, like Gecko) Chromium/25.0.1349.2 Chrome/25.0.1349.2 Safari/537.42", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36 OPR/43.0.2442.991", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36 OPR/48.0.2685.52", + "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko", + "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko", + "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)"] self.quantity = "100" self.counter = 0 + def do_search(self): @@ -18,14 +32,16 @@ class search_crtsh: urly = self.server + self.word except Exception as e: print(e) - headers = {'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:34.0) Gecko/20100101 Firefox/34.0'} try: - r=requests.get(urly,headers=headers) + params = {'User-Agent': random.choice(self.userAgent)} + r=requests.get(urly,headers=params) except Exception as e: print(e) links = self.get_info(r.text) for link in links: - r = requests.get(link, headers=headers) + params = {'User-Agent': random.choice(self.userAgent)} + r = requests.get(link, headers=params) + time.sleep(1) self.results = r.text self.totalresults += self.results