From 4fb117f3489f16adbfbf0f28bd29ca1ee18d3908 Mon Sep 17 00:00:00 2001 From: NotoriousRebel Date: Mon, 31 Dec 2018 12:20:08 -0500 Subject: [PATCH 01/11] Updated filter. --- discovery/constants.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/discovery/constants.py b/discovery/constants.py index a7478fd9..5778a33e 100644 --- a/discovery/constants.py +++ b/discovery/constants.py @@ -263,6 +263,8 @@ def filter(lst): new_lst = [] for item in lst: if (item[0].isalpha() or item[0].isdigit()) and ('xxx' not in item) and ('..' not in item): + if '252f' in item: + item = item.replace('252f', '') new_lst.append(item.lower()) return new_lst From c8fd0b5ddf7feff949fc9aee50607b0c53db9557 Mon Sep 17 00:00:00 2001 From: NotoriousRebel Date: Tue, 1 Jan 2019 21:18:09 -0500 Subject: [PATCH 02/11] Added duckduckgo as search engine, need to figure out how to crawl properly. --- discovery/duckduckgosearch.py | 88 +++++++++++++++++++++++++++++++++++ theHarvester.py | 15 +++++- 2 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 discovery/duckduckgosearch.py diff --git a/discovery/duckduckgosearch.py b/discovery/duckduckgosearch.py new file mode 100644 index 00000000..574cea1f --- /dev/null +++ b/discovery/duckduckgosearch.py @@ -0,0 +1,88 @@ +from parsers import myparser +import time +import requests +import json +from discovery.constants import * + + +class search_duckduckgo: + + def __init__(self, word, limit): + self.word = word + self.results = "" + self.totalresults = "" + self.dorks = [] + self.links = [] + self.database = "https://duckduckgo.com/?q=" + self.api = "https://api.duckduckgo.com/?q=x&format=json&pretty=1" + self.quantity = "100" + self.limit = limit + + def do_search(self): + try: # do normal scraping + url = self.api.replace('x', self.word) + headers = {'User-Agent': getUserAgent()} + r = requests.get(url, headers=headers) + except Exception as e: + print(e) + time.sleep(getDelay()) + self.results = r.text + self.totalresults += self.results + urls = self.crawl(self.results) + for url in urls: + try: + self.totalresults += requests.get(url, headers={'User-Agent': getUserAgent()}).text + time.sleep(getDelay()) + except Exception: + continue + + def crawl(self, text): + # function parses json and returns urls + urls = set() + try: + load = json.loads(text) + for key in load.keys(): + val = load.get(key) + if isinstance(val, int) or isinstance(val, dict): + continue + if isinstance(val, list): + val = val[0] + if isinstance(val, dict): + for key in val.keys(): + value = val.get(key) + if isinstance(value, str) and value != '' and 'https://' in value or 'http://' in value: + urls.add(value) + if isinstance(val, str) and val != '' and 'https://' in val or 'http://' in val: + urls.add(val) + tmp = set() + for url in urls: + if '<' in url and 'href=' in url: + equal_index = url.index('=') + true_url = '' + for ch in url[equal_index + 1:]: + if ch == '"': + tmp.add(true_url) + break + true_url += ch + else: + if url != '': + tmp.add(url) + return tmp + except Exception as e: + print(e) + return [] + + def get_emails(self): + rawres = myparser.parser(self.totalresults, self.word) + return rawres.emails() + + def get_hostnames(self): + rawres = myparser.parser(self.totalresults, self.word) + return rawres.hostnames() + + def process(self): + #while self.counter <= self.limit and self.counter <= 1000: + self.do_search() + #print("\tSearching " + str(self.counter) + " results...") + #self.counter += 100 + diff --git a/theHarvester.py b/theHarvester.py index 5340e78e..b9addfcf 100755 --- a/theHarvester.py +++ b/theHarvester.py @@ -93,7 +93,7 @@ def start(argv): elif opt == '-b': engines = set(arg.split(',')) supportedengines = set(['baidu', 'bing', 'bingapi', 'censys', 'crtsh', - 'cymon', 'dogpile', 'google', 'googleCSE', 'google-certificates', + 'cymon', 'dogpile', 'duckduckgo', 'google', 'googleCSE', 'google-certificates', 'google-profiles', 'hunter', 'linkedin', 'netcraft', 'pgp', 'securityTrails', 'threatcrowd', 'trello', 'twitter', 'vhost', 'virustotal', 'yahoo', 'all']) @@ -179,6 +179,19 @@ def start(argv): db.store_all(word, all_hosts, 'email', 'dogpile') db.store_all(word, all_hosts, 'host', 'dogpile') + elif engineitem == "duckduckgo": + print("[-] Searching in DuckDuckGo.") + from discovery import duckduckgosearch + search = duckduckgosearch.search_duckduckgo(word, limit) + search.process() + emails = filter(search.get_emails()) + hosts = filter(search.get_hostnames()) + all_hosts.extend(hosts) + all_emails.extend(emails) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'email', 'duckduckgo') + db.store_all(word, all_hosts, 'host', 'duckduckgo') + elif engineitem == "google": print("[-] Searching in Google.") search = googlesearch.search_google(word, limit, start) From 457e4a42427cf3a410480039d0bbe3dac094772e Mon Sep 17 00:00:00 2001 From: NotoriousRebel Date: Tue, 1 Jan 2019 21:38:59 -0500 Subject: [PATCH 03/11] Updated filter to remove 2Fs. --- discovery/constants.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/discovery/constants.py b/discovery/constants.py index 7ca28e30..6f9c75a9 100644 --- a/discovery/constants.py +++ b/discovery/constants.py @@ -262,9 +262,15 @@ def filter(lst): lst = set(lst) # remove duplicates new_lst = [] for item in lst: + item = str(item) if (item[0].isalpha() or item[0].isdigit()) and ('xxx' not in item) and ('..' not in item): + print('item: ', item) if '252f' in item: item = item.replace('252f', '') + if '2F' in item: + item = item.replace('2F', '') + if '2f' in item: + item = item.replace('2f', '') new_lst.append(item.lower()) return new_lst From cec034e811b1e8653d4bef89d8a705de83f76573 Mon Sep 17 00:00:00 2001 From: NotoriousRebel Date: Tue, 1 Jan 2019 21:40:36 -0500 Subject: [PATCH 04/11] Minor cosmetic fix to notify when user encounters Exception. --- discovery/duckduckgosearch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/discovery/duckduckgosearch.py b/discovery/duckduckgosearch.py index 574cea1f..e49cf6c4 100644 --- a/discovery/duckduckgosearch.py +++ b/discovery/duckduckgosearch.py @@ -69,7 +69,7 @@ class search_duckduckgo: tmp.add(url) return tmp except Exception as e: - print(e) + print('Exception occurred: ' + str(e)) return [] def get_emails(self): From 36d2acef7aa917aa5bbdd7724038482301ed0360 Mon Sep 17 00:00:00 2001 From: NotoriousRebel Date: Tue, 1 Jan 2019 21:41:59 -0500 Subject: [PATCH 05/11] Removed print statement that was used for debugging. --- discovery/constants.py | 1 - 1 file changed, 1 deletion(-) diff --git a/discovery/constants.py b/discovery/constants.py index 6f9c75a9..232f09b8 100644 --- a/discovery/constants.py +++ b/discovery/constants.py @@ -264,7 +264,6 @@ def filter(lst): for item in lst: item = str(item) if (item[0].isalpha() or item[0].isdigit()) and ('xxx' not in item) and ('..' not in item): - print('item: ', item) if '252f' in item: item = item.replace('252f', '') if '2F' in item: From 5c64cb38f8e05dfab4e3bc7dd707d079bc9d900e Mon Sep 17 00:00:00 2001 From: NotoriousRebel Date: Tue, 1 Jan 2019 21:45:58 -0500 Subject: [PATCH 06/11] Added docstring. --- discovery/duckduckgosearch.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/discovery/duckduckgosearch.py b/discovery/duckduckgosearch.py index e49cf6c4..3ba49d28 100644 --- a/discovery/duckduckgosearch.py +++ b/discovery/duckduckgosearch.py @@ -14,7 +14,7 @@ class search_duckduckgo: self.dorks = [] self.links = [] self.database = "https://duckduckgo.com/?q=" - self.api = "https://api.duckduckgo.com/?q=x&format=json&pretty=1" + self.api = "https://api.duckduckgo.com/?q=x&format=json&pretty=1" # currently using api self.quantity = "100" self.limit = limit @@ -37,17 +37,21 @@ class search_duckduckgo: continue def crawl(self, text): - # function parses json and returns urls + """ + function parses json and returns urls + :param text: formatted json + :return: set of urls + """ urls = set() try: load = json.loads(text) - for key in load.keys(): + for key in load.keys(): # iterate through keys of dict val = load.get(key) if isinstance(val, int) or isinstance(val, dict): continue if isinstance(val, list): - val = val[0] - if isinstance(val, dict): + val = val[0] # first value should be dict + if isinstance(val, dict): # sanity check for key in val.keys(): value = val.get(key) if isinstance(value, str) and value != '' and 'https://' in value or 'http://' in value: @@ -56,7 +60,7 @@ class search_duckduckgo: urls.add(val) tmp = set() for url in urls: - if '<' in url and 'href=' in url: + if '<' in url and 'href=' in url: # format is equal_index = url.index('=') true_url = '' for ch in url[equal_index + 1:]: @@ -81,8 +85,4 @@ class search_duckduckgo: return rawres.hostnames() def process(self): - #while self.counter <= self.limit and self.counter <= 1000: - self.do_search() - #print("\tSearching " + str(self.counter) + " results...") - #self.counter += 100 - + self.do_search() # only need to search once since using API From d684e22171be7ea54f8cd2b5b2ab7357399b3601 Mon Sep 17 00:00:00 2001 From: NotoriousRebel Date: Tue, 1 Jan 2019 23:33:18 -0500 Subject: [PATCH 07/11] Made user agent for initial request to duckduckgo api constant. --- discovery/DNS/Base.py | 2 +- discovery/duckduckgosearch.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/discovery/DNS/Base.py b/discovery/DNS/Base.py index c0306c93..0ac18fc8 100644 --- a/discovery/DNS/Base.py +++ b/discovery/DNS/Base.py @@ -242,7 +242,7 @@ class DnsRequest: class DnsAsyncRequest(DnsRequest, asyncore.dispatcher_with_send): - " an asynchronous request object. out of date, probably broken " + " an asynchronous request object. out of date, probably broken " def __init__(self, *name, **args): DnsRequest.__init__(self, *name, **args) diff --git a/discovery/duckduckgosearch.py b/discovery/duckduckgosearch.py index 3ba49d28..3c48b0c0 100644 --- a/discovery/duckduckgosearch.py +++ b/discovery/duckduckgosearch.py @@ -21,7 +21,7 @@ class search_duckduckgo: def do_search(self): try: # do normal scraping url = self.api.replace('x', self.word) - headers = {'User-Agent': getUserAgent()} + headers = {'User-Agent': googleUA} r = requests.get(url, headers=headers) except Exception as e: print(e) @@ -74,6 +74,8 @@ class search_duckduckgo: return tmp except Exception as e: print('Exception occurred: ' + str(e)) + import traceback as t + print(t.print_exc()) return [] def get_emails(self): @@ -85,4 +87,4 @@ class search_duckduckgo: return rawres.hostnames() def process(self): - self.do_search() # only need to search once since using API + self.do_search() # only need to search once since using API From 23260d9aef39c667466c5e6ea1411f4535fced6e Mon Sep 17 00:00:00 2001 From: NotoriousRebel Date: Fri, 4 Jan 2019 17:22:16 -0500 Subject: [PATCH 08/11] Resyncing --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3df79f16..6ab1e7c3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ beautifulsoup4==4.7.0 plotly==3.4.2 -requests==2.21.0 \ No newline at end of file +requests==2.21.0 From e03dc69118a55292592ca1dd72259803867c7ce0 Mon Sep 17 00:00:00 2001 From: NotoriousRebel Date: Fri, 4 Jan 2019 21:55:02 -0500 Subject: [PATCH 09/11] Syncing. --- requirements.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 8fc56f88..adb75b15 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,5 @@ beautifulsoup4>=4.7.0 plotly>=3.4.2 requests>=2.21.0 -<<<<<<< HEAD -======= texttable>=1.4.0 shodan>=1.10.0 ->>>>>>> 25b922d700ff5ecdaaa8a82e77c5e3ef8377cbda From c089699d6adfe3c4da89d3b226968825ed16481c Mon Sep 17 00:00:00 2001 From: NotoriousRebel Date: Fri, 4 Jan 2019 21:58:01 -0500 Subject: [PATCH 10/11] Updated dockerfile to use python 3.6 --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 07a9b6c4..d0802046 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ -FROM python:2-alpine +FROM python:3.6-alpine RUN mkdir /app RUN pip install requests beautifulsoup4 texttable plotly shodan WORKDIR /app COPY . /app RUN chmod +x *.py - ENTRYPOINT ["/app/theHarvester.py"] +ENTRYPOINT ["/app/theHarvester.py"] From 208025b23b1201d1c5d6d541d491548f507863e4 Mon Sep 17 00:00:00 2001 From: Matt <36310667+NotoriousRebel@users.noreply.github.com> Date: Sat, 5 Jan 2019 11:47:42 -0500 Subject: [PATCH 11/11] Update Base.py Fixed space. --- discovery/DNS/Base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/discovery/DNS/Base.py b/discovery/DNS/Base.py index 0ac18fc8..c0306c93 100644 --- a/discovery/DNS/Base.py +++ b/discovery/DNS/Base.py @@ -242,7 +242,7 @@ class DnsRequest: class DnsAsyncRequest(DnsRequest, asyncore.dispatcher_with_send): - " an asynchronous request object. out of date, probably broken " + " an asynchronous request object. out of date, probably broken " def __init__(self, *name, **args): DnsRequest.__init__(self, *name, **args)