WEBDIENSTE/theHarvester
1
0
Fork 0
mirror of https://github.com/laramies/theHarvester.git synced 2026-09-17 23:27:46 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #28 from NotoriousRebel/dev

Browse Source 
Ported github-code
This commit is contained in:
Matt
2020-01-02 21:13:42 -05:00
committed by GitHub
parent 207ff7a081 3c5221f324
commit 19b10effb7
17 changed files with 123 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/theHarvester.yml
+19 -19
View File
@@ -29,78 +29,78 @@ jobs:
- name: Run theHarvester module baidu
run: |
python theHarvester.py -d apple.com -b baidu
python theHarvester.py -d yale.edu -b baidu
- name: Run theHarvester module bing
run: |
python theHarvester.py -d apple.com -b bing
python theHarvester.py -d yale.edu -b bing
- name: Run theHarvester module certspotter
run: |
python theHarvester.py -d apple.com -b certspotter
python theHarvester.py -d yale.edu -b certspotter
- name: Run theHarvester module crtsh
run: |
python theHarvester.py -d apple.com -b crtsh
python theHarvester.py -d yale.edu -b crtsh
- name: Run theHarvester module dnsdumpster
run: |
python theHarvester.py -d apple.com -b dnsdumpster
python theHarvester.py -d yale.edu -b dnsdumpster
- name: Run theHarvester module dogplie
run: |
python theHarvester.py -d apple.com -b dogpile
python theHarvester.py -d yale.edu -b dogpile
- name: Run theHarvester module duckduckgo
run: |
python theHarvester.py -d apple.com -b duckduckgo
python theHarvester.py -d yale.edu -b duckduckgo
- name: Run theHarvester module exalead
run: |
python theHarvester.py -d apple.com -b exalead
python theHarvester.py -d yale.edu -b exalead
- name: Run theHarvester module google
run: |
python theHarvester.py -d apple.com -b google
python theHarvester.py -d yale.edu -b google
- name: Run theHarvester module Intelx
run: |
python theHarvester.py -d apple.com -b intelx
python theHarvester.py -d yale.edu -b intelx
- name: Run theHarvester module linkedin
run: |
python theHarvester.py -d apple.com -b linkedin
python theHarvester.py -d yale.edu -b linkedin
- name: Run theHarvester module linkedin_links
run: |
python theHarvester.py -d apple.com -b linkedin_links
python theHarvester.py -d yale.edu -b linkedin_links
- name: Run theHarvester module netcraft
run: |
python theHarvester.py -d apple.com -b netcraft
python theHarvester.py -d yale.edu -b netcraft
- name: Run theHarvester module Otx
run: |
python theHarvester.py -d apple.com -b otx
python theHarvester.py -d yale.edu -b otx
- name: Run theHarvester module threatcrowd
run: |
python theHarvester.py -d apple.com -b threatcrowd
python theHarvester.py -d yale.edu -b threatcrowd
- name: Run theHarvester module trello
run: |
python theHarvester.py -d apple.com -b trello
python theHarvester.py -d yale.edu -b trello
- name: Run theHarvester module twitter
run: |
python theHarvester.py -d apple.com -b twitter
python theHarvester.py -d yale.edu -b twitter
- name: Run theHarvester module virustotal
run: |
python theHarvester.py -d apple.com -b virustotal
python theHarvester.py -d yale.edu -b virustotal
- name: Run theHarvester module yahoo
run: |
python theHarvester.py -d apple.com -b yahoo
python theHarvester.py -d yale.edu -b yahoo
- name: Lint with flake8
run: |
.gitignore
+2 -1
View File
@@ -10,4 +10,5 @@ venv
.pytest_cache
build/
dist/
theHarvester.egg-info/
theHarvester.egg-info/
api-keys.yaml
theHarvester/discovery/baidusearch.py
+2 -2
View File
@@ -27,8 +27,8 @@ class SearchBaidu:
async def get_emails(self):
rawres = myparser.Parser(self.total_results, self.word)
return rawres.emails()
return await rawres.emails()
async def get_hostnames(self):
rawres = myparser.Parser(self.total_results, self.word)
return rawres.hostnames()
return await rawres.hostnames()
theHarvester/discovery/bingsearch.py
+3 -4
View File
@@ -1,7 +1,6 @@
from theHarvester.discovery.constants import *
from theHarvester.lib.core import *
from theHarvester.parsers import myparser
from theHarvester.lib.core import AsyncFetcher
class SearchBing:
@@ -58,15 +57,15 @@ class SearchBing:
async def get_emails(self):
rawres = myparser.Parser(self.total_results, self.word)
return rawres.emails()
return await rawres.emails()
async def get_hostnames(self):
rawres = myparser.Parser(self.total_results, self.word)
return rawres.hostnames()
return await rawres.hostnames()
async def get_allhostnames(self):
rawres = myparser.Parser(self.total_results, self.word)
return rawres.hostnames_all()
return await rawres.hostnames_all()
async def process(self, api):
if api == 'yes':
theHarvester/discovery/dnsdumpster.py
+1 -1
View File
@@ -38,7 +38,7 @@ class SearchDnsDumpster:
async def get_hostnames(self):
rawres = myparser.Parser(self.totalresults, self.word)
return rawres.hostnames()
return await rawres.hostnames()
async def process(self):
await self.do_search() # Only need to do it once.
theHarvester/discovery/dogpilesearch.py
+2 -2
View File
@@ -28,8 +28,8 @@ class SearchDogpile:
async def get_emails(self):
rawres = myparser.Parser(self.total_results, self.word)
return rawres.emails()
return await rawres.emails()
async def get_hostnames(self):
rawres = myparser.Parser(self.total_results, self.word)
return rawres.hostnames()
return await rawres.hostnames()
theHarvester/discovery/duckduckgosearch.py
+2 -2
View File
@@ -73,11 +73,11 @@ class SearchDuckDuckGo:
async def get_emails(self):
rawres = myparser.Parser(self.totalresults, self.word)
return rawres.emails()
return await rawres.emails()
async def get_hostnames(self):
rawres = myparser.Parser(self.totalresults, self.word)
return rawres.hostnames()
return await rawres.hostnames()
async def process(self):
await self.do_search() # Only need to search once since using API.
theHarvester/discovery/exaleadsearch.py
+3 -3
View File
@@ -52,15 +52,15 @@ class SearchExalead:
async def get_emails(self):
rawres = myparser.Parser(self.total_results, self.word)
return rawres.emails()
return await rawres.emails()
async def get_hostnames(self):
rawres = myparser.Parser(self.total_results, self.word)
return rawres.hostnames()
return await rawres.hostnames()
async def get_files(self):
rawres = myparser.Parser(self.total_results, self.word)
return rawres.fileurls(self.files)
return await rawres.fileurls(self.files)
async def process(self):
print('Searching results')
theHarvester/discovery/githubcode.py
+50 -45
View File
@@ -1,10 +1,9 @@
from theHarvester.discovery.constants import *
from theHarvester.lib.core import *
from theHarvester.parsers import myparser
import requests
from requests import Response
import time
from typing import List, Dict, Any, Optional, NamedTuple
from typing import List, Dict, Any, Optional, NamedTuple, Tuple
import asyncio
import aiohttp
import urllib.parse as urlparse
@@ -40,20 +39,21 @@ class SearchGithubCode:
raise MissingKey(True)
@staticmethod
def fragments_from_response(response: Response) -> List[str]:
items: List[Dict[str, Any]] = response.json().get('items') or list()
async def fragments_from_response(json_data: dict) -> List[str]:
items: List[Dict[str, Any]] = json_data.get('items') or list()
fragments: List[str] = list()
for item in items:
matches = item.get("text_matches") or list()
for match in matches:
fragments.append(match.get("fragment"))
return [fragment for fragment in fragments if fragment is not None]
@staticmethod
def page_from_response(page: str, response: Response) -> Optional[Any]:
page_link = response.links.get(page)
async def page_from_response(page: str, links) -> Optional[Any]:
page_link = links.get(page)
if page_link:
parsed = urlparse.urlparse(page_link.get("url"))
parsed = urlparse.urlparse(str(page_link.get("url")))
params = urlparse.parse_qs(parsed.query)
pages: List[Any] = params.get('page', [None])
page_number = pages[0] and int(pages[0])
@@ -61,21 +61,22 @@ class SearchGithubCode:
else:
return None
def handle_response(self, response: Response) -> Optional[Any]:
if response.ok:
results = self.fragments_from_response(response)
next_page = self.page_from_response("next", response)
last_page = self.page_from_response("last", response)
async def handle_response(self, response: Tuple[str, dict, int, Any]):
text, json_data, status, links = response
if status == 200:
results = await self.fragments_from_response(json_data)
next_page = await self.page_from_response("next", links)
last_page = await self.page_from_response("last", links)
return SuccessResult(results, next_page, last_page)
elif response.status_code == 429 or response.status_code == 403:
elif status == 429 or status == 403:
return RetryResult(60)
else:
try:
return ErrorResult(response.status_code, response.json())
return ErrorResult(status, json_data)
except ValueError:
return ErrorResult(response.status_code, response.text)
return ErrorResult(status, text)
def do_search(self, page: Optional[int]) -> Response:
async def do_search(self, page: Optional[int]) -> Tuple[str, dict, int, Any]:
if page is None:
url = f'https://{self.server}/search/code?q="{self.word}"'
else:
@@ -84,42 +85,46 @@ class SearchGithubCode:
'Host': self.server,
'User-agent': Core.get_user_agent(),
'Accept': "application/vnd.github.v3.text-match+json",
'Authorization': 'token {}'.format(self.key)
'Authorization': f'token {self.key}'
}
return requests.get(url=url, headers=headers, verify=True)
async with aiohttp.ClientSession(headers=headers) as sess:
async with sess.get(url) as resp:
return await resp.text(), await resp.json(), resp.status, resp.links
@staticmethod
def next_page_or_end(result: SuccessResult) -> Optional[int]:
async def next_page_or_end(result: SuccessResult) -> Optional[int]:
if result.next_page is not None:
return result.next_page
else:
return result.last_page
def process(self):
while self.counter <= self.limit and self.page is not None:
api_response = self.do_search(self.page)
result = self.handle_response(api_response)
if type(result) == SuccessResult:
print(f'\tSearching {self.counter} results.')
for fragment in result.fragments:
self.total_results += fragment
self.counter = self.counter + 1
async def process(self):
try:
while self.counter <= self.limit and self.page is not None:
api_response = await self.do_search(self.page)
result = await self.handle_response(api_response)
if type(result) == SuccessResult:
print(f'\tSearching {self.counter} results.')
for fragment in result.fragments:
self.total_results += fragment
self.counter = self.counter + 1
self.page = await self.next_page_or_end(result)
await asyncio.sleep(getDelay())
elif type(result) == RetryResult:
sleepy_time = getDelay() + result.time
print(f'\tRetrying page in {sleepy_time} seconds...')
await asyncio.sleep(sleepy_time)
elif type(result) == ErrorResult:
raise Exception(f"\tException occurred: status_code: {result.status_code} reason: {result.body}")
else:
raise Exception("\tUnknown exception occurred")
except Exception as e:
print(f'An exception has occurred: {e}')
self.page = self.next_page_or_end(result)
time.sleep(getDelay())
elif type(result) == RetryResult:
sleepy_time = getDelay() + result.time
print(f'\tRetrying page in {sleepy_time} seconds...')
time.sleep(sleepy_time)
elif type(result) == ErrorResult:
raise Exception(f"\tException occurred: status_code: {result.status_code} reason: {result.body}")
else:
raise Exception("\tUnknown exception occurred")
def get_emails(self):
async def get_emails(self):
rawres = myparser.Parser(self.total_results, self.word)
return rawres.emails()
return await rawres.emails()
def get_hostnames(self):
async def get_hostnames(self):
rawres = myparser.Parser(self.total_results, self.word)
return rawres.hostnames()
return await rawres.hostnames()
theHarvester/discovery/huntersearch.py
+3 -3
View File
@@ -25,12 +25,12 @@ class SearchHunter:
async def get_emails(self):
rawres = myparser.Parser(self.total_results, self.word)
return rawres.emails()
return await rawres.emails()
async def get_hostnames(self):
rawres = myparser.Parser(self.total_results, self.word)
return rawres.hostnames()
return await rawres.hostnames()
async def get_profiles(self):
rawres = myparser.Parser(self.total_results, self.word)
return rawres.profiles()
return await rawres.profiles()
theHarvester/discovery/netcraft.py
+1 -1
View File
@@ -91,7 +91,7 @@ class SearchNetcraft:
async def get_hostnames(self):
rawres = myparser.Parser(self.totalresults, self.word)
return rawres.hostnames()
return await rawres.hostnames()
async def process(self):
await self.do_search()
theHarvester/discovery/securitytrailssearch.py
-1
View File
@@ -53,4 +53,3 @@ class SearchSecuritytrail:
async def get_hostnames(self) -> set:
return self.info[1]
theHarvester/discovery/threatcrowd.py
+2 -1
View File
@@ -1,3 +1,4 @@
from typing import Coroutine
from theHarvester.lib.core import *
from theHarvester.parsers import myparser
@@ -19,7 +20,7 @@ class SearchThreatcrowd:
print(e)
self.totalresults += self.results
async def get_hostnames(self) -> set:
async def get_hostnames(self) -> Coroutine:
return myparser.Parser(self.results, self.word).hostnames()
async def process(self):
theHarvester/discovery/twittersearch.py
+8 -9
View File
@@ -14,21 +14,20 @@ class SearchTwitter:
self.limit = int(limit)
self.counter = 0
def do_search(self):
async def do_search(self):
base_url = f'https://{self.server}/search?num=100&start=xx&hl=en&meta=&q=site%3Atwitter.com%20intitle%3A%22on+Twitter%22%20{self.word}'
headers = {'User-Agent': Core.get_user_agent()}
try:
urls = [base_url.replace("xx", str(num)) for num in range(0, self.limit, 10) if num <= self.limit]
request = (grequests.get(url, headers=headers) for url in urls)
response = grequests.imap(request, size=5)
for entry in response:
self.totalresults += entry.content.decode('UTF-8')
responses = await AsyncFetcher.fetch_all(urls, headers=headers)
for response in responses:
self.totalresults += response
except Exception as error:
print(error)
def get_people(self):
async def get_people(self):
rawres = myparser.Parser(self.totalresults, self.word)
to_parse = rawres.people_twitter()
to_parse = await rawres.people_twitter()
# fix invalid handles that look like @user other_output
handles = set()
for handle in to_parse:
@@ -37,5 +36,5 @@ class SearchTwitter:
handles.add(result.group(0))
return handles
def process(self):
self.do_search()
async def process(self):
await self.do_search()
theHarvester/discovery/virustotal.py
+1 -1
View File
@@ -20,7 +20,7 @@ class SearchVirustotal:
async def get_hostnames(self):
rawres = myparser.Parser(self.results, self.word)
return rawres.hostnames()
return await rawres.hostnames()
async def process(self):
print('\tSearching results.')
theHarvester/discovery/yahoosearch.py
+2 -2
View File
@@ -26,7 +26,7 @@ class SearchYahoo:
async def get_emails(self):
rawres = myparser.Parser(self.total_results, self.word)
toparse_emails = rawres.emails()
toparse_emails = await rawres.emails()
emails = set()
# strip out numbers and dashes for emails that look like xxx-xxx-xxxemail@host.tld
for email in toparse_emails:
@@ -39,4 +39,4 @@ class SearchYahoo:
async def get_hostnames(self):
rawres = myparser.Parser(self.total_results, self.word)
return rawres.hostnames()
return await rawres.hostnames()
theHarvester/parsers/myparser.py
+22 -22
View File
@@ -8,7 +8,7 @@ class Parser:
self.word = word
self.temp = []
def genericClean(self):
async def genericClean(self):
self.results = self.results.replace('<em>', '').replace('<b>', '').replace('</b>', '').replace('</em>', '')\
.replace('%2f', '').replace('%3a', '').replace('<strong>', '').replace('</strong>', '')\
.replace('<wbr>', '').replace('</wbr>', '')
@@ -16,28 +16,28 @@ class Parser:
for search in ('<', '>', ':', '=', ';', '&', '%3A', '%3D', '%3C', '/', '\\'):
self.results = self.results.replace(search, ' ')
def urlClean(self):
async def urlClean(self):
self.results = self.results.replace('<em>', '').replace('</em>', '').replace('%2f', '').replace('%3a', '')
for search in ('<', '>', ':', '=', ';', '&', '%3A', '%3D', '%3C'):
self.results = self.results.replace(search, ' ')
def emails(self):
self.genericClean()
async def emails(self):
await self.genericClean()
# Local part is required, charset is flexible.
# https://tools.ietf.org/html/rfc6531 (removed * and () as they provide FP mostly)
reg_emails = re.compile(r'[a-zA-Z0-9.\-_+#~!$&\',;=:]+' + '@' + '[a-zA-Z0-9.-]*' + self.word.replace('www.', ''))
self.temp = reg_emails.findall(self.results)
emails = self.unique()
emails = await self.unique()
true_emails = {str(email)[1:].lower().strip() if len(str(email)) > 1 and str(email)[0] == '.'
else len(str(email)) > 1 and str(email).lower().strip() for email in emails}
# if email starts with dot shift email string and make sure all emails are lowercase
return true_emails
def fileurls(self, file):
async def fileurls(self, file):
urls = []
reg_urls = re.compile('<a href="(.*?)"')
self.temp = reg_urls.findall(self.results)
allurls = self.unique()
allurls = await self.unique()
for iteration in allurls:
if iteration.count('webcache') or iteration.count('google.com') or iteration.count('search?hl'):
pass
@@ -45,17 +45,17 @@ class Parser:
urls.append(iteration)
return urls
def hostnames(self):
self.genericClean()
async def hostnames(self):
await self.genericClean()
reg_hosts = re.compile(r'[a-zA-Z0-9.-]*\.' + self.word)
self.temp = reg_hosts.findall(self.results)
hostnames = self.unique()
hostnames = await self.unique()
reg_hosts = re.compile(r'[a-zA-Z0-9.-]*\.' + self.word.replace('www.', ''))
self.temp = reg_hosts.findall(self.results)
hostnames.extend(self.unique())
hostnames.extend(await self.unique())
return list(set(hostnames))
def people_googleplus(self):
async def people_googleplus(self):
self.results = re.sub('</b>', '', self.results)
self.results = re.sub('<b>', '', self.results)
reg_people = re.compile(r'>[a-zA-Z0-9._ ]* - Google\+')
@@ -71,7 +71,7 @@ class Parser:
resul.append(delete)
return resul
def hostnames_all(self):
async def hostnames_all(self):
reg_hosts = re.compile('<cite>(.*?)</cite>')
temp = reg_hosts.findall(self.results)
for iteration in temp:
@@ -80,10 +80,10 @@ class Parser:
else:
res = iteration.split('/')[0]
self.temp.append(res)
hostnames = self.unique()
hostnames = await self.unique()
return hostnames
def links_linkedin(self):
async def links_linkedin(self):
reg_links = re.compile(r"url=https:\/\/www\.linkedin.com(.*?)&")
self.temp = reg_links.findall(self.results)
resul = []
@@ -92,7 +92,7 @@ class Parser:
resul.append("https://www.linkedin.com" + final_url)
return resul
def people_linkedin(self):
async def people_linkedin(self):
reg_people = re.compile(r'">[a-zA-Z0-9._ -]* \| LinkedIn')
self.temp = reg_people.findall(self.results)
resul = []
@@ -106,10 +106,10 @@ class Parser:
resul.append(delete)
return resul
def people_twitter(self):
async def people_twitter(self):
reg_people = re.compile(r'(@[a-zA-Z0-9._ -]*)')
self.temp = reg_people.findall(self.results)
users = self.unique()
users = await self.unique()
resul = []
for iteration in users:
delete = iteration.replace(' | LinkedIn', '')
@@ -121,7 +121,7 @@ class Parser:
resul.append(delete)
return resul
def profiles(self):
async def profiles(self):
reg_people = re.compile(r'">[a-zA-Z0-9._ -]* - <em>Google Profile</em>')
self.temp = reg_people.findall(self.results)
resul = []
@@ -133,7 +133,7 @@ class Parser:
resul.append(delete)
return resul
def set(self):
async def set(self):
reg_sets = re.compile(r'>[a-zA-Z0-9]*</a></font>')
self.temp = reg_sets.findall(self.results)
sets = []
@@ -143,10 +143,10 @@ class Parser:
sets.append(delete)
return sets
def urls(self):
async def urls(self):
found = re.finditer(r'(http|https)://(www\.)?trello.com/([a-zA-Z0-9\-_\.]+/?)*', self.results)
urls = {match.group().strip() for match in found}
return urls
def unique(self) -> list:
async def unique(self) -> list:
return list(set(self.temp))