mirror of
https://github.com/laramies/theHarvester.git
synced 2026-09-12 04:37:40 +02:00
Merge pull request #149 from jzold/master
Shodan integration to theHarvester
This commit is contained in:
Christian Martorella
+1
-1
@@ -1,6 +1,6 @@
|
||||
FROM python:2-alpine
|
||||
RUN mkdir /app
|
||||
RUN pip install requests beautifulsoup4 texttable
|
||||
RUN pip install requests beautifulsoup4 texttable plotly shodan
|
||||
WORKDIR /app
|
||||
COPY . /app
|
||||
RUN chmod +x *.py
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
from discovery.shodan.api import WebAPI
|
||||
from discovery.shodan.client import Shodan
|
||||
from discovery.shodan.exception import APIError
|
||||
@@ -1,9 +0,0 @@
|
||||
class Alert:
|
||||
def __init__(self):
|
||||
self.id = None
|
||||
self.name = None
|
||||
self.api_key = None
|
||||
self.filters = None
|
||||
self.credits = None
|
||||
self.created = None
|
||||
self.expires = None
|
||||
@@ -1,222 +0,0 @@
|
||||
try:
|
||||
# Python 2
|
||||
from urllib2 import urlopen
|
||||
from urllib import urlencode
|
||||
except:
|
||||
# Python 3
|
||||
from urllib.request import urlopen
|
||||
from urllib.parse import urlencode
|
||||
|
||||
#from json import loads #TODO FIX
|
||||
|
||||
from .exception import WebAPIError
|
||||
|
||||
|
||||
__all__ = ['WebAPI']
|
||||
|
||||
|
||||
class WebAPI:
|
||||
"""Wrapper around the SHODAN webservices API"""
|
||||
|
||||
class Exploits:
|
||||
|
||||
def __init__(self, parent):
|
||||
self.parent = parent
|
||||
|
||||
def search(self, query, sources=[], cve=None, osvdb=None, msb=None, bid=None):
|
||||
"""Search the entire Shodan Exploits archive using the same query syntax
|
||||
as the website.
|
||||
|
||||
Arguments:
|
||||
query -- exploit search query; same syntax as website
|
||||
|
||||
Optional arguments:
|
||||
sources -- metasploit, cve, osvdb, exploitdb
|
||||
cve -- CVE identifier (ex. 2010-0432)
|
||||
osvdb -- OSVDB identifier (ex. 11666)
|
||||
msb -- Microsoft Security Bulletin ID (ex. MS05-030)
|
||||
bid -- Bugtraq identifier (ex. 13951)
|
||||
|
||||
"""
|
||||
if sources:
|
||||
query += ' source:%s' % (','.join(sources))
|
||||
if cve:
|
||||
query += ' cve:%s' % (str(cve).strip())
|
||||
if osvdb:
|
||||
query += ' osvdb:%s' % (str(osvdb).strip())
|
||||
if msb:
|
||||
query += ' msb:%s' % (str(msb).strip())
|
||||
if bid:
|
||||
query += ' bid:%s' % (str(bid).strip())
|
||||
return self.parent._request('api', {'q': query}, service='exploits')
|
||||
|
||||
class ExploitDb:
|
||||
|
||||
def __init__(self, parent):
|
||||
self.parent = parent
|
||||
|
||||
def download(self, id):
|
||||
"""DEPRECATED
|
||||
Download the exploit code from the ExploitDB archive.
|
||||
|
||||
Arguments:
|
||||
id -- ID of the ExploitDB entry
|
||||
"""
|
||||
query = '_id:%s' % id
|
||||
return self.parent.search(query, sources=['exploitdb'])
|
||||
|
||||
def search(self, query, **kwargs):
|
||||
"""Search the ExploitDB archive.
|
||||
|
||||
Arguments:
|
||||
query -- Search terms
|
||||
|
||||
Returns:
|
||||
A dictionary with 2 main items: matches (list) and total (int).
|
||||
"""
|
||||
return self.parent.search(query, sources=['exploitdb'])
|
||||
|
||||
|
||||
class Msf:
|
||||
|
||||
def __init__(self, parent):
|
||||
self.parent = parent
|
||||
|
||||
def download(self, id):
|
||||
"""Download a metasploit module given the fullname (id) of it.
|
||||
|
||||
Arguments:
|
||||
id -- fullname of the module (ex. auxiliary/admin/backupexec/dump)
|
||||
|
||||
Returns:
|
||||
A dictionary with the following fields:
|
||||
filename -- Name of the file
|
||||
content-type -- Mimetype
|
||||
data -- File content
|
||||
"""
|
||||
query = '_id:%s' % id
|
||||
return self.parent.search(query, sources=['metasploit'])
|
||||
|
||||
def search(self, query, **kwargs):
|
||||
"""Search for a Metasploit module.
|
||||
"""
|
||||
return self.parent.search(query, sources=['metasploit'])
|
||||
|
||||
def __init__(self, key):
|
||||
"""Initializes the API object.
|
||||
|
||||
Arguments:
|
||||
key -- your API key
|
||||
|
||||
"""
|
||||
print('WARNING: This class is deprecated, please upgrade to use "shodan.Shodan()" instead of shodan.WebAPI()')
|
||||
self.api_key = key
|
||||
self.base_url = 'http://www.shodanhq.com/api/'
|
||||
self.base_exploits_url = 'https://exploits.shodan.io/'
|
||||
self.exploits = self.Exploits(self)
|
||||
self.exploitdb = self.ExploitDb(self.exploits)
|
||||
self.msf = self.Msf(self.exploits)
|
||||
|
||||
def _request(self, function, params, service='shodan'):
|
||||
"""General-purpose function to create web requests to SHODAN.
|
||||
|
||||
Arguments:
|
||||
function -- name of the function you want to execute
|
||||
params -- dictionary of parameters for the function
|
||||
|
||||
Returns
|
||||
A JSON string containing the function's results.
|
||||
|
||||
"""
|
||||
# Add the API key parameter automatically
|
||||
params['key'] = self.api_key
|
||||
|
||||
# Determine the base_url based on which service we're interacting with
|
||||
base_url = {
|
||||
'shodan': self.base_url,
|
||||
'exploits': self.base_exploits_url,
|
||||
}.get(service, 'shodan')
|
||||
|
||||
# Send the request
|
||||
try:
|
||||
data = urlopen(base_url + function + '?' + urlencode(params)).read().decode('utf-8')
|
||||
except:
|
||||
raise WebAPIError('Unable to connect to Shodan')
|
||||
|
||||
# Parse the text from JSON to a dict
|
||||
data = loads(data)
|
||||
|
||||
# Raise an exception if an error occurred
|
||||
if data.get('error', None):
|
||||
raise WebAPIError(data['error'])
|
||||
|
||||
# Return the data
|
||||
return data
|
||||
|
||||
def count(self, query):
|
||||
"""Returns the total number of search results for the query.
|
||||
"""
|
||||
return self._request('count', {'q': query})
|
||||
|
||||
def locations(self, query):
|
||||
"""Return a break-down of all the countries and cities that the results for
|
||||
the given search are located in.
|
||||
"""
|
||||
return self._request('locations', {'q': query})
|
||||
|
||||
def fingerprint(self, banner):
|
||||
"""Determine the software based on the banner.
|
||||
|
||||
Arguments:
|
||||
banner - HTTP banner
|
||||
|
||||
Returns:
|
||||
A list of software that matched the given banner.
|
||||
"""
|
||||
return self._request('fingerprint', {'banner': banner})
|
||||
|
||||
def host(self, ip):
|
||||
"""Get all available information on an IP.
|
||||
|
||||
Arguments:
|
||||
ip -- IP of the computer
|
||||
|
||||
Returns:
|
||||
All available information SHODAN has on the given IP,
|
||||
subject to API key restrictions.
|
||||
|
||||
"""
|
||||
return self._request('host', {'ip': ip})
|
||||
|
||||
def info(self):
|
||||
"""Returns information about the current API key, such as a list of add-ons
|
||||
and other features that are enabled for the current user's API plan.
|
||||
"""
|
||||
return self._request('info', {})
|
||||
|
||||
def search(self, query, page=1, limit=None, offset=None):
|
||||
"""Search the SHODAN database.
|
||||
|
||||
Arguments:
|
||||
query -- search query; identical syntax to the website
|
||||
|
||||
Optional arguments:
|
||||
page -- page number of the search results
|
||||
limit -- number of results to return
|
||||
offset -- search offset to begin getting results from
|
||||
|
||||
Returns:
|
||||
A dictionary with 3 main items: matches, countries and total.
|
||||
Visit the website for more detailed information.
|
||||
|
||||
"""
|
||||
args = {
|
||||
'q': query,
|
||||
'p': page,
|
||||
}
|
||||
if limit:
|
||||
args['l'] = limit
|
||||
if offset:
|
||||
args['o'] = offset
|
||||
|
||||
return self._request('search', args)
|
||||
@@ -1,5 +0,0 @@
|
||||
from .csvc import CsvConverter
|
||||
from .excel import ExcelConverter
|
||||
from .geojson import GeoJsonConverter
|
||||
from .images import ImagesConverter
|
||||
from .kml import KmlConverter
|
||||
@@ -1,8 +0,0 @@
|
||||
|
||||
class Converter:
|
||||
|
||||
def __init__(self, fout):
|
||||
self.fout = fout
|
||||
|
||||
def process(self, fout):
|
||||
pass
|
||||
@@ -1,87 +0,0 @@
|
||||
|
||||
from .base import Converter
|
||||
from ...helpers import iterate_files
|
||||
|
||||
from collections import MutableMapping
|
||||
from csv import writer as csv_writer, excel
|
||||
|
||||
|
||||
class CsvConverter(Converter):
|
||||
|
||||
fields = [
|
||||
'data',
|
||||
'hostnames',
|
||||
'ip',
|
||||
'ip_str',
|
||||
'ipv6',
|
||||
'org',
|
||||
'isp',
|
||||
'location.country_code',
|
||||
'location.city',
|
||||
'location.country_name',
|
||||
'location.latitude',
|
||||
'location.longitude',
|
||||
'os',
|
||||
'asn',
|
||||
'port',
|
||||
'transport',
|
||||
'product',
|
||||
'version',
|
||||
|
||||
'ssl.cipher.version',
|
||||
'ssl.cipher.bits',
|
||||
'ssl.cipher.name',
|
||||
'ssl.alpn',
|
||||
'ssl.versions',
|
||||
'ssl.cert.serial',
|
||||
'ssl.cert.fingerprint.sha1',
|
||||
'ssl.cert.fingerprint.sha256',
|
||||
|
||||
'html',
|
||||
'title',
|
||||
]
|
||||
|
||||
def process(self, files):
|
||||
writer = csv_writer(self.fout, dialect=excel)
|
||||
|
||||
# Write the header
|
||||
writer.writerow(self.fields)
|
||||
|
||||
for banner in iterate_files(files):
|
||||
try:
|
||||
row = []
|
||||
for field in self.fields:
|
||||
value = self.banner_field(banner, field)
|
||||
row.append(value)
|
||||
writer.writerow(row)
|
||||
except:
|
||||
pass
|
||||
|
||||
def banner_field(self, banner, flat_field):
|
||||
# The provided field is a collapsed form of the actual field
|
||||
fields = flat_field.split('.')
|
||||
|
||||
try:
|
||||
current_obj = banner
|
||||
for field in fields:
|
||||
current_obj = current_obj[field]
|
||||
|
||||
# Convert a list into a concatenated string
|
||||
if isinstance(current_obj, list):
|
||||
current_obj = ','.join([str(i) for i in current_obj])
|
||||
|
||||
return current_obj
|
||||
except:
|
||||
pass
|
||||
|
||||
return ''
|
||||
|
||||
def flatten(self, d, parent_key='', sep='.'):
|
||||
items = []
|
||||
for k, v in d.items():
|
||||
new_key = parent_key + sep + k if parent_key else k
|
||||
if isinstance(v, MutableMapping):
|
||||
items.extend(flatten(v, new_key, sep=sep).items())
|
||||
else:
|
||||
items.append((new_key, v))
|
||||
return dict(items)
|
||||
@@ -1,130 +0,0 @@
|
||||
|
||||
from .base import Converter
|
||||
from ...helpers import iterate_files, get_ip
|
||||
|
||||
from collections import defaultdict, MutableMapping
|
||||
from xlsxwriter import Workbook
|
||||
|
||||
|
||||
class ExcelConverter(Converter):
|
||||
|
||||
fields = [
|
||||
'port',
|
||||
'timestamp',
|
||||
'data',
|
||||
'hostnames',
|
||||
'org',
|
||||
'isp',
|
||||
'location.country_name',
|
||||
'location.country_code',
|
||||
'location.city',
|
||||
'os',
|
||||
'asn',
|
||||
'transport',
|
||||
'product',
|
||||
'version',
|
||||
|
||||
'http.server',
|
||||
'http.title',
|
||||
]
|
||||
|
||||
field_names = {
|
||||
'org': 'Organization',
|
||||
'isp': 'ISP',
|
||||
'location.country_code': 'Country ISO Code',
|
||||
'location.country_name': 'Country',
|
||||
'location.city': 'City',
|
||||
'os': 'OS',
|
||||
'asn': 'ASN',
|
||||
|
||||
'http.server': 'Web Server',
|
||||
'http.title': 'Website Title',
|
||||
}
|
||||
|
||||
def process(self, files):
|
||||
# Get the filename from the already-open file handle
|
||||
filename = self.fout.name
|
||||
|
||||
# Close the existing file as the XlsxWriter library handles that for us
|
||||
self.fout.close()
|
||||
|
||||
# Create the new workbook
|
||||
workbook = Workbook(filename)
|
||||
|
||||
# Define some common styles/ formats
|
||||
bold = workbook.add_format({
|
||||
'bold': 1,
|
||||
})
|
||||
|
||||
# Create the main worksheet where all the raw data is shown
|
||||
main_sheet = workbook.add_worksheet('Raw Data')
|
||||
|
||||
# Write the header
|
||||
main_sheet.write(0, 0, 'IP', bold) # The IP field can be either ip_str or ipv6 so we treat it differently
|
||||
main_sheet.set_column(0, 0, 20)
|
||||
|
||||
row = 0
|
||||
col = 1
|
||||
for field in self.fields:
|
||||
name = self.field_names.get(field, field.capitalize())
|
||||
main_sheet.write(row, col, name, bold)
|
||||
col += 1
|
||||
row += 1
|
||||
|
||||
total = 0
|
||||
ports = defaultdict(int)
|
||||
for banner in iterate_files(files):
|
||||
try:
|
||||
# Build the list that contains all the relevant values
|
||||
data = []
|
||||
for field in self.fields:
|
||||
value = self.banner_field(banner, field)
|
||||
data.append(value)
|
||||
|
||||
# Write those values to the main workbook
|
||||
# Starting off w/ the special "IP" property
|
||||
main_sheet.write_string(row, 0, get_ip(banner))
|
||||
col = 1
|
||||
|
||||
for value in data:
|
||||
main_sheet.write(row, col, value)
|
||||
col += 1
|
||||
row += 1
|
||||
except:
|
||||
pass
|
||||
|
||||
# Aggregate summary information
|
||||
total += 1
|
||||
ports[banner['port']] += 1
|
||||
|
||||
summary_sheet = workbook.add_worksheet('Summary')
|
||||
summary_sheet.write(0, 0, 'Total', bold)
|
||||
summary_sheet.write(0, 1, total)
|
||||
|
||||
# Ports Distribution
|
||||
summary_sheet.write(0, 3, 'Ports Distribution', bold)
|
||||
row = 1
|
||||
col = 3
|
||||
for key, value in sorted(ports.items(), reverse=True, key=lambda kv: (kv[1], kv[0])):
|
||||
summary_sheet.write(row, col, key)
|
||||
summary_sheet.write(row, col + 1, value)
|
||||
row += 1
|
||||
|
||||
def banner_field(self, banner, flat_field):
|
||||
# The provided field is a collapsed form of the actual field
|
||||
fields = flat_field.split('.')
|
||||
|
||||
try:
|
||||
current_obj = banner
|
||||
for field in fields:
|
||||
current_obj = current_obj[field]
|
||||
|
||||
# Convert a list into a concatenated string
|
||||
if isinstance(current_obj, list):
|
||||
current_obj = ','.join([str(i) for i in current_obj])
|
||||
|
||||
return current_obj
|
||||
except:
|
||||
pass
|
||||
|
||||
return ''
|
||||
@@ -1,57 +0,0 @@
|
||||
|
||||
from base import Converter
|
||||
from ...helpers import get_ip, iterate_files
|
||||
|
||||
class GeoJsonConverter(Converter):
|
||||
|
||||
def header(self):
|
||||
self.fout.write("""{
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
""")
|
||||
|
||||
def footer(self):
|
||||
self.fout.write("""{ }]}""")
|
||||
|
||||
def process(self, files):
|
||||
# Write the header
|
||||
self.header()
|
||||
|
||||
hosts = {}
|
||||
for banner in iterate_files(files):
|
||||
ip = get_ip(banner)
|
||||
if not ip:
|
||||
continue
|
||||
|
||||
if ip not in hosts:
|
||||
hosts[ip] = banner
|
||||
hosts[ip]['ports'] = []
|
||||
|
||||
hosts[ip]['ports'].append(banner['port'])
|
||||
|
||||
for ip, host in iter(hosts.items()):
|
||||
self.write(host)
|
||||
|
||||
self.footer()
|
||||
|
||||
|
||||
def write(self, host):
|
||||
try:
|
||||
ip = get_ip(host)
|
||||
lat, lon = host['location']['latitude'], host['location']['longitude']
|
||||
|
||||
feature = """{
|
||||
"type": "Feature",
|
||||
"id": "{}",
|
||||
"properties": {
|
||||
"name": "{}"
|
||||
},
|
||||
"geometry": {
|
||||
"type": "Point",
|
||||
"coordinates": [{}, {}]
|
||||
}
|
||||
}""".format(ip, ip, lat, lon)
|
||||
|
||||
self.fout.write(feature)
|
||||
except Exception as e:
|
||||
pass
|
||||
@@ -1,51 +0,0 @@
|
||||
|
||||
from base import Converter
|
||||
from ...helpers import iterate_files, get_ip, get_screenshot
|
||||
|
||||
# Needed for decoding base64-strings in Python3
|
||||
from codecs import decode
|
||||
|
||||
import os
|
||||
|
||||
|
||||
class ImagesConverter(Converter):
|
||||
|
||||
# The Images converter is special in that it creates a directory and there's
|
||||
# special code in the Shodan CLI that relies on the "dirname" property to let
|
||||
# the user know where the images have been stored.
|
||||
dirname = None
|
||||
|
||||
def process(self, files):
|
||||
# Get the filename from the already-open file handle and use it as
|
||||
# the directory name to store the images.
|
||||
self.dirname = self.fout.name[:-7] + '-images'
|
||||
|
||||
# Remove the original file that was created
|
||||
self.fout.close()
|
||||
os.unlink(self.fout.name)
|
||||
|
||||
# Create the directory if it doesn't yet exist
|
||||
if not os.path.exists(self.dirname):
|
||||
os.mkdir(self.dirname)
|
||||
|
||||
# Close the existing file as the XlsxWriter library handles that for us
|
||||
self.fout.close()
|
||||
|
||||
# Loop through all the banners in the data file
|
||||
for banner in iterate_files(files):
|
||||
screenshot = get_screenshot(banner)
|
||||
if screenshot:
|
||||
filename = '{}/{}-{}'.format(self.dirname, get_ip(banner), banner['port'])
|
||||
|
||||
# If a file with the name already exists then count up until we
|
||||
# create a new, unique filename
|
||||
counter = 0
|
||||
tmpname = filename
|
||||
while os.path.exists(tmpname + '.jpg'):
|
||||
tmpname = '{}-{}'.format(filename, counter)
|
||||
counter += 1
|
||||
filename = tmpname + '.jpg'
|
||||
|
||||
fout = open(filename, 'wb')
|
||||
fout.write(decode(screenshot['data'].encode(), 'base64'))
|
||||
fout.close()
|
||||
@@ -1,127 +0,0 @@
|
||||
|
||||
from .base import Converter
|
||||
from ...helpers import iterate_files
|
||||
|
||||
class KmlConverter(Converter):
|
||||
|
||||
def header(self):
|
||||
self.fout.write("""<?xml version="1.0" encoding="UTF-8"?>
|
||||
<kml xmlns="http://www.opengis.net/kml/2.2">
|
||||
<Document>""")
|
||||
|
||||
def footer(self):
|
||||
self.fout.write("""</Document></kml>""")
|
||||
|
||||
def process(self, files):
|
||||
# Write the header
|
||||
self.header()
|
||||
|
||||
hosts = {}
|
||||
for banner in iterate_files(files):
|
||||
ip = banner.get('ip_str', banner.get('ipv6', None))
|
||||
if not ip:
|
||||
continue
|
||||
|
||||
if ip not in hosts:
|
||||
hosts[ip] = banner
|
||||
hosts[ip]['ports'] = []
|
||||
|
||||
hosts[ip]['ports'].append(banner['port'])
|
||||
|
||||
for ip, host in iter(hosts.items()):
|
||||
self.write(host)
|
||||
|
||||
self.footer()
|
||||
|
||||
|
||||
def write(self, host):
|
||||
try:
|
||||
ip = host.get('ip_str', host.get('ipv6', None))
|
||||
lat, lon = host['location']['latitude'], host['location']['longitude']
|
||||
|
||||
placemark = '<Placemark><name><![CDATA[<h1 style="margin-bottom:0;padding-bottom:0;font-size:1.5em">{}</h1>]]></name>'.format(ip)
|
||||
placemark += '<description><![CDATA['
|
||||
|
||||
if 'hostnames' in host and host['hostnames']:
|
||||
placemark += '<div><a style="color: #999;margin-top:-10px;padding-top:0;" href="http://{0}" target="_blank">{0}</a></div>'.format(host['hostnames'][0])
|
||||
|
||||
test = """
|
||||
<table>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>City</td>
|
||||
<th>Albuquerque</th>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Country</td>
|
||||
<th>United States</th>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Organization</td>
|
||||
<th>Nexcess.net L.L.C.</th>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
<h2>Ports</h2>
|
||||
<ul>
|
||||
"""
|
||||
|
||||
placemark += '<h2>Ports</h2><ul>'
|
||||
|
||||
for port in host['ports']:
|
||||
placemark += """
|
||||
<li style="background-color: #1CA8DD;
|
||||
color: #FFF;
|
||||
float: left;
|
||||
font-family: Arial;
|
||||
font-size: 12px;
|
||||
font-weight: bold;
|
||||
height: 48px;
|
||||
margin: 5px;
|
||||
position: relative;
|
||||
text-shadow: none;
|
||||
width: 48px;"><span style="color: #FFF;
|
||||
height: 34px;
|
||||
position: absolute;
|
||||
text-align: center;
|
||||
top: 30%;
|
||||
width: 48px;">{}</span>
|
||||
</li>
|
||||
""".format(port)
|
||||
|
||||
placemark += '</ul><div style="clear:both"></div>'
|
||||
|
||||
placemark += """
|
||||
<div style="text-align:center"><a href="https://www.shodan.io/host/{0}" style="display: inline-block;
|
||||
padding: 4px 10px;
|
||||
margin-bottom: 0px;
|
||||
font-size: 13px;
|
||||
line-height: 18px;
|
||||
color: #333;
|
||||
text-align: center;
|
||||
text-shadow: 0px 1px 1px rgba(255, 255, 255, 0.75);
|
||||
vertical-align: middle;
|
||||
cursor: pointer;
|
||||
background-color: #F5F5F5;
|
||||
background-image: -moz-linear-gradient(center top , #FFF, #E6E6E6);
|
||||
background-repeat: repeat-x;
|
||||
border-width: 1px;
|
||||
border-style: solid;
|
||||
border-color: #CCC #CCC #B3B3B3;
|
||||
-moz-border-top-colors: none;
|
||||
-moz-border-right-colors: none;
|
||||
-moz-border-bottom-colors: none;
|
||||
-moz-border-left-colors: none;
|
||||
border-image: none;
|
||||
border-radius: 4px;
|
||||
box-shadow: 0px 1px 0px rgba(255, 255, 255, 0.2) inset, 0px 1px 2px rgba(0, 0, 0, 0.05);" target="_blank">View Details</a></div>
|
||||
<div>powered by <a href="https://www.shodan.io" target="_blank">Shodan</a></div>
|
||||
""".format(ip)
|
||||
|
||||
placemark += ']]></description>'
|
||||
placemark += '<Point><coordinates>{},{}</coordinates></Point>'.format(lon, lat)
|
||||
placemark += '</Placemark>'
|
||||
|
||||
self.fout.write(placemark.encode('utf-8'))
|
||||
except Exception as e:
|
||||
pass
|
||||
@@ -1,23 +0,0 @@
|
||||
'''
|
||||
Helper methods to create your own CLI commands.
|
||||
'''
|
||||
import click
|
||||
import os
|
||||
|
||||
from .settings import SHODAN_CONFIG_DIR
|
||||
|
||||
def get_api_key():
|
||||
'''Returns the API key of the current logged-in user.'''
|
||||
shodan_dir = os.path.expanduser(SHODAN_CONFIG_DIR)
|
||||
keyfile = shodan_dir + '/api_key'
|
||||
|
||||
# If the file doesn't yet exist let the user know that they need to
|
||||
# initialize the shodan cli
|
||||
if not os.path.exists(keyfile):
|
||||
raise click.ClickException('Please run "shodan init <api key>" before using this command')
|
||||
|
||||
# Make sure it is a read-only file
|
||||
os.chmod(keyfile, 0o600)
|
||||
|
||||
with open(keyfile, 'r') as fin:
|
||||
return fin.read().strip()
|
||||
@@ -1,10 +0,0 @@
|
||||
|
||||
SHODAN_CONFIG_DIR = '~/.shodan/'
|
||||
COLORIZE_FIELDS = {
|
||||
'ip_str': 'green',
|
||||
'port': 'yellow',
|
||||
'data': 'white',
|
||||
'hostnames': 'magenta',
|
||||
'org': 'cyan',
|
||||
'vulns': 'red',
|
||||
}
|
||||
@@ -1,251 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
'''
|
||||
F-Secure Virus World Map console edition
|
||||
|
||||
See README.md for more details
|
||||
|
||||
Copyright 2012-2013 Jyrki Muukkonen
|
||||
|
||||
Released under the MIT license.
|
||||
See LICENSE.txt or http://www.opensource.org/licenses/mit-license.php
|
||||
|
||||
ASCII map in map-world-01.txt is copyright:
|
||||
"Map 1998 Matthew Thomas. Freely usable as long as this line is included"
|
||||
|
||||
'''
|
||||
import curses
|
||||
import locale
|
||||
import random
|
||||
import time
|
||||
|
||||
from discovery.shodan.helpers import get_ip
|
||||
|
||||
|
||||
MAPS = {
|
||||
'world': {
|
||||
# offset (as (y, x) for curses...)
|
||||
'corners': (1, 4, 23, 73),
|
||||
# lat top, lon left, lat bottom, lon right
|
||||
'coords': [90.0, -180.0, -90.0, 180.0],
|
||||
'data': '''
|
||||
. _..::__: ,-"-"._ |7 , _,.__
|
||||
_.___ _ _<_>`!(._`.`-. / _._ `_ ,_/ ' '-._.---.-.__
|
||||
.{ " " `-==,',._\{ \ / {) / _ ">_,-' ` mt-2_
|
||||
\_.:--. `._ )`^-. "' , [_/( __,/-'
|
||||
'"' \ " _L oD_,--' ) /. (|
|
||||
| ,' _)_.\\._<> 6 _,' / '
|
||||
`. / [_/_'` `"( <'} )
|
||||
\\ .-. ) / `-'"..' `:._ _) '
|
||||
` \ ( `( / `:\ > \ ,-^. /' '
|
||||
`._, "" | \`' \| ?_) {\
|
||||
`=.---. `._._ ,' "` |' ,- '.
|
||||
| `-._ | / `:`<_|h--._
|
||||
( > . | , `=.__.`-'\
|
||||
`. / | |{| ,-.,\ .
|
||||
| ,' \ / `' ," \
|
||||
| / |_' | __ /
|
||||
| | '-' `-' \.
|
||||
|/ " /
|
||||
\. '
|
||||
|
||||
,/ ______._.--._ _..---.---------._
|
||||
,-----"-..?----_/ ) _,-'" " (
|
||||
Map 1998 Matthew Thomas. Freely usable as long as this line is included
|
||||
'''
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class AsciiMap(object):
|
||||
"""
|
||||
Helper class for handling map drawing and coordinate calculations
|
||||
"""
|
||||
def __init__(self, map_name='world', map_conf=None, window=None, encoding=None):
|
||||
if map_conf is None:
|
||||
map_conf = MAPS[map_name]
|
||||
self.map = map_conf['data']
|
||||
self.coords = map_conf['coords']
|
||||
self.corners = map_conf['corners']
|
||||
if window is None:
|
||||
window = curses.newwin(0, 0)
|
||||
self.window = window
|
||||
|
||||
self.data = []
|
||||
self.data_timestamp = None
|
||||
|
||||
# JSON contents _should_ be UTF8 (so, python internal unicode here...)
|
||||
if encoding is None:
|
||||
encoding = locale.getpreferredencoding()
|
||||
self.encoding = encoding
|
||||
|
||||
# check if we can use transparent background or not
|
||||
if curses.can_change_color():
|
||||
curses.use_default_colors()
|
||||
background = -1
|
||||
else:
|
||||
background = curses.COLOR_BLACK
|
||||
|
||||
tmp_colors = [
|
||||
('red', curses.COLOR_RED, background),
|
||||
('blue', curses.COLOR_BLUE, background),
|
||||
('pink', curses.COLOR_MAGENTA, background)
|
||||
]
|
||||
|
||||
self.colors = {}
|
||||
if curses.has_colors():
|
||||
for i, (name, fgcolor, bgcolor) in enumerate(tmp_colors, 1):
|
||||
curses.init_pair(i, fgcolor, bgcolor)
|
||||
self.colors[name] = i
|
||||
|
||||
def latlon_to_coords(self, lat, lon):
|
||||
"""
|
||||
Convert lat/lon coordinates to character positions.
|
||||
Very naive version, assumes that we are drawing the whole world
|
||||
TODO: filter out stuff that doesn't fit
|
||||
TODO: make it possible to use "zoomed" maps
|
||||
"""
|
||||
width = (self.corners[3]-self.corners[1])
|
||||
height = (self.corners[2]-self.corners[0])
|
||||
|
||||
# change to 0-180, 0-360
|
||||
abs_lat = -lat+90
|
||||
abs_lon = lon+180
|
||||
x = (abs_lon/360.0)*width + self.corners[1]
|
||||
y = (abs_lat/180.0)*height + self.corners[0]
|
||||
return int(x), int(y)
|
||||
|
||||
def set_data(self, data):
|
||||
"""
|
||||
Set / convert internal data.
|
||||
For now it just selects a random set to show.
|
||||
"""
|
||||
entries = []
|
||||
|
||||
# Grab 5 random banners to display
|
||||
for banner in random.sample(data, min(len(data), 5)):
|
||||
desc = '{} -> {} / {}'.format(get_ip(banner), banner['port'], banner['location']['country_code'])
|
||||
if banner['location']['city']:
|
||||
desc += ' {}'.format(banner['location']['city'])
|
||||
|
||||
if 'tags' in banner and banner['tags']:
|
||||
desc += ' / {}'.format(','.join(banner['tags']))
|
||||
|
||||
entry = (
|
||||
float(banner['location']['latitude']),
|
||||
float(banner['location']['longitude']),
|
||||
'*',
|
||||
desc,
|
||||
curses.A_BOLD,
|
||||
'red',
|
||||
)
|
||||
entries.append(entry)
|
||||
self.data = entries
|
||||
|
||||
def draw(self, target):
|
||||
""" Draw internal data to curses window """
|
||||
self.window.clear()
|
||||
self.window.addstr(0, 0, self.map)
|
||||
|
||||
# FIXME: position to be defined in map config?
|
||||
row = self.corners[2]-6
|
||||
items_to_show = 5
|
||||
for lat, lon, char, desc, attrs, color in self.data:
|
||||
# to make this work almost everywhere. see http://docs.python.org/2/library/curses.html
|
||||
if desc:
|
||||
desc = desc.encode(self.encoding, 'ignore')
|
||||
if items_to_show <= 0:
|
||||
break
|
||||
char_x, char_y = self.latlon_to_coords(lat, lon)
|
||||
if self.colors and color:
|
||||
attrs |= curses.color_pair(self.colors[color])
|
||||
self.window.addstr(char_y, char_x, char, attrs)
|
||||
if desc:
|
||||
det_show = "%s %s" % (char, desc)
|
||||
else:
|
||||
det_show = None
|
||||
|
||||
if det_show is not None:
|
||||
try:
|
||||
self.window.addstr(row, 1, det_show, attrs)
|
||||
row += 1
|
||||
items_to_show -= 1
|
||||
except Exception:
|
||||
# FIXME: check window size before addstr()
|
||||
break
|
||||
self.window.overwrite(target)
|
||||
self.window.leaveok(1)
|
||||
|
||||
|
||||
class MapApp(object):
|
||||
""" Virus World Map ncurses application """
|
||||
def __init__(self, api):
|
||||
self.api = api
|
||||
self.data = None
|
||||
self.last_fetch = 0
|
||||
self.sleep = 10 # tenths of seconds, for curses.halfdelay()
|
||||
self.polling_interval = 60
|
||||
|
||||
def fetch_data(self, epoch_now, force_refresh=False):
|
||||
""" (Re)fetch data from JSON stream """
|
||||
refresh = False
|
||||
if force_refresh or self.data is None:
|
||||
refresh = True
|
||||
else:
|
||||
if self.last_fetch + self.polling_interval <= epoch_now:
|
||||
refresh = True
|
||||
|
||||
if refresh:
|
||||
try:
|
||||
# Grab 20 banners from the main stream
|
||||
banners = []
|
||||
for banner in self.api.stream.banners():
|
||||
if 'location' in banner and banner['location']['latitude']:
|
||||
banners.append(banner)
|
||||
if len(banners) >= 20:
|
||||
break
|
||||
self.data = banners
|
||||
self.last_fetch = epoch_now
|
||||
except Exception:
|
||||
raise
|
||||
return refresh
|
||||
|
||||
def run(self, scr):
|
||||
""" Initialize and run the application """
|
||||
m = AsciiMap()
|
||||
curses.halfdelay(self.sleep)
|
||||
while True:
|
||||
now = int(time.time())
|
||||
refresh = self.fetch_data(now)
|
||||
m.set_data(self.data)
|
||||
m.draw(scr)
|
||||
scr.addstr(0, 1, 'Shodan Radar', curses.A_BOLD)
|
||||
scr.addstr(0, 40, time.strftime("%c UTC", time.gmtime(now)).rjust(37), curses.A_BOLD)
|
||||
|
||||
# Key Input
|
||||
# q - Quit
|
||||
event = scr.getch()
|
||||
if event == ord('q'):
|
||||
break
|
||||
|
||||
# redraw window (to fix encoding/rendering bugs and to hide other messages to same tty)
|
||||
# user pressed 'r' or new data was fetched
|
||||
if refresh:
|
||||
m.window.redrawwin()
|
||||
|
||||
|
||||
def launch_map(api):
|
||||
app = MapApp(api)
|
||||
return curses.wrapper(app.run)
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
""" Main function / entry point """
|
||||
from discovery.shodan import Shodan
|
||||
from discovery.shodan.cli.helpers import get_api_key
|
||||
|
||||
api = Shodan(get_api_key())
|
||||
return launch_map(api)
|
||||
|
||||
if __name__ == '__main__':
|
||||
import sys
|
||||
sys.exit(main())
|
||||
@@ -1,508 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
shodan.client
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
This module implements the Shodan API.
|
||||
|
||||
:copyright: (c) 2014- by John Matherly
|
||||
"""
|
||||
import time
|
||||
|
||||
import requests
|
||||
import json
|
||||
|
||||
from .exception import APIError
|
||||
from .helpers import api_request, create_facet_string
|
||||
from .stream import Stream
|
||||
|
||||
|
||||
# Try to disable the SSL warnings in urllib3 since not everybody can install
|
||||
# C extensions. If you're able to install C extensions you can try to run:
|
||||
#
|
||||
# pip install requests[security]
|
||||
#
|
||||
# Which will download libraries that offer more full-featured SSL classes
|
||||
try:
|
||||
requests.packages.urllib3.disable_warnings()
|
||||
except:
|
||||
pass
|
||||
|
||||
# Define a basestring type if necessary for Python3 compatibility
|
||||
try:
|
||||
str
|
||||
except NameError:
|
||||
basestring = str
|
||||
|
||||
|
||||
class Shodan:
|
||||
"""Wrapper around the Shodan REST and Streaming APIs
|
||||
|
||||
:param key: The Shodan API key that can be obtained from your account page (https://account.shodan.io)
|
||||
:type key: str
|
||||
:ivar exploits: An instance of `shodan.Shodan.Exploits` that provides access to the Exploits REST API.
|
||||
:ivar stream: An instance of `shodan.Shodan.Stream` that provides access to the Streaming API.
|
||||
"""
|
||||
|
||||
class Data:
|
||||
|
||||
def __init__(self, parent):
|
||||
self.parent = parent
|
||||
|
||||
def list_datasets(self):
|
||||
"""Returns a list of datasets that the user has permission to download.
|
||||
|
||||
:returns: A list of objects where every object describes a dataset
|
||||
"""
|
||||
return self.parent._request('/shodan/data', {})
|
||||
|
||||
def list_files(self, dataset):
|
||||
"""Returns a list of files that belong to the given dataset.
|
||||
|
||||
:returns: A list of objects where each object contains a 'name', 'size', 'timestamp' and 'url'
|
||||
"""
|
||||
return self.parent._request('/shodan/data/{}'.format(dataset), {})
|
||||
|
||||
class Tools:
|
||||
|
||||
def __init__(self, parent):
|
||||
self.parent = parent
|
||||
|
||||
def myip(self):
|
||||
"""Get your current IP address as seen from the Internet.
|
||||
|
||||
:returns: str -- your IP address
|
||||
"""
|
||||
return self.parent._request('/tools/myip', {})
|
||||
|
||||
class Exploits:
|
||||
|
||||
def __init__(self, parent):
|
||||
self.parent = parent
|
||||
|
||||
def search(self, query, page=1, facets=None):
|
||||
"""Search the entire Shodan Exploits archive using the same query syntax
|
||||
as the website.
|
||||
|
||||
:param query: The exploit search query; same syntax as website.
|
||||
:type query: str
|
||||
:param facets: A list of strings or tuples to get summary information on.
|
||||
:type facets: str
|
||||
:param page: The page number to access.
|
||||
:type page: int
|
||||
:returns: dict -- a dictionary containing the results of the search.
|
||||
"""
|
||||
query_args = {
|
||||
'query': query,
|
||||
'page': page,
|
||||
}
|
||||
if facets:
|
||||
query_args['facets'] = create_facet_string(facets)
|
||||
|
||||
return self.parent._request('/api/search', query_args, service='exploits')
|
||||
|
||||
def count(self, query, facets=None):
|
||||
"""Search the entire Shodan Exploits archive but only return the total # of results,
|
||||
not the actual exploits.
|
||||
|
||||
:param query: The exploit search query; same syntax as website.
|
||||
:type query: str
|
||||
:param facets: A list of strings or tuples to get summary information on.
|
||||
:type facets: str
|
||||
:returns: dict -- a dictionary containing the results of the search.
|
||||
|
||||
"""
|
||||
query_args = {
|
||||
'query': query,
|
||||
}
|
||||
if facets:
|
||||
query_args['facets'] = create_facet_string(facets)
|
||||
|
||||
return self.parent._request('/api/count', query_args, service='exploits')
|
||||
|
||||
class Labs:
|
||||
|
||||
def __init__(self, parent):
|
||||
self.parent = parent
|
||||
|
||||
def honeyscore(self, ip):
|
||||
"""Calculate the probability of an IP being an ICS honeypot.
|
||||
|
||||
:param ip: IP address of the device
|
||||
:type ip: str
|
||||
|
||||
:returns: int -- honeyscore ranging from 0.0 to 1.0
|
||||
"""
|
||||
return self.parent._request('/labs/honeyscore/{}'.format(ip), {})
|
||||
|
||||
def __init__(self, key):
|
||||
"""Initializes the API object.
|
||||
|
||||
:param key: The Shodan API key.
|
||||
:type key: str
|
||||
"""
|
||||
self.api_key = key
|
||||
self.base_url = 'https://api.shodan.io'
|
||||
self.base_exploits_url = 'https://exploits.shodan.io'
|
||||
self.data = self.Data(self)
|
||||
self.exploits = self.Exploits(self)
|
||||
self.labs = self.Labs(self)
|
||||
self.tools = self.Tools(self)
|
||||
self.stream = Stream(key)
|
||||
self._session = requests.Session()
|
||||
|
||||
def _request(self, function, params, service='shodan', method='get'):
|
||||
"""General-purpose function to create web requests to SHODAN.
|
||||
|
||||
Arguments:
|
||||
function -- name of the function you want to execute
|
||||
params -- dictionary of parameters for the function
|
||||
|
||||
Returns
|
||||
A dictionary containing the function's results.
|
||||
|
||||
"""
|
||||
# Add the API key parameter automatically
|
||||
params['key'] = self.api_key
|
||||
|
||||
# Determine the base_url based on which service we're interacting with
|
||||
base_url = {
|
||||
'shodan': self.base_url,
|
||||
'exploits': self.base_exploits_url,
|
||||
}.get(service, 'shodan')
|
||||
|
||||
# Send the request
|
||||
try:
|
||||
if method.lower() == 'post':
|
||||
data = self._session.post(base_url + function, params)
|
||||
else:
|
||||
data = self._session.get(base_url + function, params=params)
|
||||
except:
|
||||
raise APIError('Unable to connect to Shodan')
|
||||
|
||||
# Check that the API key wasn't rejected
|
||||
if data.status_code == 401:
|
||||
try:
|
||||
# Return the actual error message if the API returned valid JSON
|
||||
error = data.json()['error']
|
||||
except Exception as e:
|
||||
error = 'Invalid API key'
|
||||
|
||||
raise APIError(error)
|
||||
|
||||
# Parse the text into JSON
|
||||
try:
|
||||
data = data.json()
|
||||
except:
|
||||
raise APIError('Unable to parse JSON response')
|
||||
|
||||
# Raise an exception if an error occurred
|
||||
if type(data) == dict and 'error' in data:
|
||||
raise APIError(data['error'])
|
||||
|
||||
# Return the data
|
||||
return data
|
||||
|
||||
def count(self, query, facets=None):
|
||||
"""Returns the total number of search results for the query.
|
||||
|
||||
:param query: Search query; identical syntax to the website
|
||||
:type query: str
|
||||
:param facets: (optional) A list of properties to get summary information on
|
||||
:type facets: str
|
||||
|
||||
:returns: A dictionary with 1 main property: total. If facets have been provided then another property called "facets" will be available at the top-level of the dictionary. Visit the website for more detailed information.
|
||||
"""
|
||||
query_args = {
|
||||
'query': query,
|
||||
}
|
||||
if facets:
|
||||
query_args['facets'] = create_facet_string(facets)
|
||||
return self._request('/shodan/host/count', query_args)
|
||||
|
||||
def host(self, ips, history=False, minify=False):
|
||||
"""Get all available information on an IP.
|
||||
|
||||
:param ip: IP of the computer
|
||||
:type ip: str
|
||||
:param history: (optional) True if you want to grab the historical (non-current) banners for the host, False otherwise.
|
||||
:type history: bool
|
||||
:param minify: (optional) True to only return the list of ports and the general host information, no banners, False otherwise.
|
||||
:type minify: bool
|
||||
"""
|
||||
if isinstance(ips, str):
|
||||
ips = [ips]
|
||||
|
||||
params = {}
|
||||
if history:
|
||||
params['history'] = history
|
||||
if minify:
|
||||
params['minify'] = minify
|
||||
return self._request('/shodan/host/%s' % ','.join(ips), params)
|
||||
|
||||
def info(self):
|
||||
"""Returns information about the current API key, such as a list of add-ons
|
||||
and other features that are enabled for the current user's API plan.
|
||||
"""
|
||||
return self._request('/api-info', {})
|
||||
|
||||
def ports(self):
|
||||
"""Get a list of ports that Shodan crawls
|
||||
|
||||
:returns: An array containing the ports that Shodan crawls for.
|
||||
"""
|
||||
return self._request('/shodan/ports', {})
|
||||
|
||||
def protocols(self):
|
||||
"""Get a list of protocols that the Shodan on-demand scanning API supports.
|
||||
|
||||
:returns: A dictionary containing the protocol name and description.
|
||||
"""
|
||||
return self._request('/shodan/protocols', {})
|
||||
|
||||
def scan(self, ips, force=False):
|
||||
"""Scan a network using Shodan
|
||||
|
||||
:param ips: A list of IPs or netblocks in CIDR notation or an object structured like:
|
||||
{
|
||||
"9.9.9.9": [
|
||||
(443, "https"),
|
||||
(8080, "http")
|
||||
],
|
||||
"1.1.1.0/24": [
|
||||
(503, "modbus")
|
||||
]
|
||||
}
|
||||
:type ips: str or dict
|
||||
:param force: Whether or not to force Shodan to re-scan the provided IPs. Only available to enterprise users.
|
||||
:type force: bool
|
||||
|
||||
:returns: A dictionary with a unique ID to check on the scan progress, the number of IPs that will be crawled and how many scan credits are left.
|
||||
"""
|
||||
if isinstance(ips, basestring):
|
||||
ips = [ips]
|
||||
|
||||
if isinstance(ips, dict):
|
||||
networks = json.dumps(ips)
|
||||
else:
|
||||
networks = ','.join(ips)
|
||||
|
||||
params = {
|
||||
'ips': networks,
|
||||
'force': force,
|
||||
}
|
||||
|
||||
return self._request('/shodan/scan', params, method='post')
|
||||
|
||||
def scan_internet(self, port, protocol):
|
||||
"""Scan a network using Shodan
|
||||
|
||||
:param port: The port that should get scanned.
|
||||
:type port: int
|
||||
:param port: The name of the protocol as returned by the protocols() method.
|
||||
:type port: str
|
||||
|
||||
:returns: A dictionary with a unique ID to check on the scan progress.
|
||||
"""
|
||||
params = {
|
||||
'port': port,
|
||||
'protocol': protocol,
|
||||
}
|
||||
|
||||
return self._request('/shodan/scan/internet', params, method='post')
|
||||
|
||||
def scan_status(self, scan_id):
|
||||
"""Get the status information about a previously submitted scan.
|
||||
|
||||
:param id: The unique ID for the scan that was submitted
|
||||
:type id: str
|
||||
|
||||
:returns: A dictionary with general information about the scan, including its status in getting processed.
|
||||
"""
|
||||
return self._request('/shodan/scan/%s' % scan_id, {})
|
||||
|
||||
def search(self, query, page=1, limit=None, offset=None, facets=None, minify=True):
|
||||
"""Search the SHODAN database.
|
||||
|
||||
:param query: Search query; identical syntax to the website
|
||||
:type query: str
|
||||
:param page: (optional) Page number of the search results
|
||||
:type page: int
|
||||
:param limit: (optional) Number of results to return
|
||||
:type limit: int
|
||||
:param offset: (optional) Search offset to begin getting results from
|
||||
:type offset: int
|
||||
:param facets: (optional) A list of properties to get summary information on
|
||||
:type facets: str
|
||||
:param minify: (optional) Whether to minify the banner and only return the important data
|
||||
:type minify: bool
|
||||
|
||||
:returns: A dictionary with 2 main items: matches and total. If facets have been provided then another property called "facets" will be available at the top-level of the dictionary. Visit the website for more detailed information.
|
||||
"""
|
||||
args = {
|
||||
'query': query,
|
||||
'minify': minify,
|
||||
}
|
||||
if limit:
|
||||
args['limit'] = limit
|
||||
if offset:
|
||||
args['offset'] = offset
|
||||
else:
|
||||
args['page'] = page
|
||||
|
||||
if facets:
|
||||
args['facets'] = create_facet_string(facets)
|
||||
|
||||
return self._request('/shodan/host/search', args)
|
||||
|
||||
def search_cursor(self, query, minify=True, retries=5):
|
||||
"""Search the SHODAN database.
|
||||
|
||||
This method returns an iterator that can directly be in a loop. Use it when you want to loop over
|
||||
all of the results of a search query. But this method doesn't return a "matches" array or the "total"
|
||||
information. And it also can't be used with facets, it's only use is to iterate over results more
|
||||
easily.
|
||||
|
||||
:param query: Search query; identical syntax to the website
|
||||
:type query: str
|
||||
:param minify: (optional) Whether to minify the banner and only return the important data
|
||||
:type minify: bool
|
||||
:param retries: (optional) How often to retry the search in case it times out
|
||||
:type minify: int
|
||||
|
||||
:returns: A search cursor that can be used as an iterator/ generator.
|
||||
"""
|
||||
args = {
|
||||
'query': query,
|
||||
'minify': minify,
|
||||
}
|
||||
|
||||
page = 1
|
||||
tries = 0
|
||||
while page == 1 or results['matches']:
|
||||
try:
|
||||
results = self.search(query, minify=minify, page=page)
|
||||
for banner in results['matches']:
|
||||
try:
|
||||
yield banner
|
||||
except GeneratorExit:
|
||||
return # exit out of the function
|
||||
page += 1
|
||||
tries = 0
|
||||
except:
|
||||
# We've retried several times but it keeps failing, so lets error out
|
||||
if tries >= retries:
|
||||
break
|
||||
|
||||
tries += 1
|
||||
time.sleep(1.0) # wait 1 second if the search errored out for some reason
|
||||
|
||||
def search_tokens(self, query):
|
||||
"""Returns information about the search query itself (filters used etc.)
|
||||
|
||||
:param query: Search query; identical syntax to the website
|
||||
:type query: str
|
||||
|
||||
:returns: A dictionary with 4 main properties: filters, errors, attributes and string.
|
||||
"""
|
||||
query_args = {
|
||||
'query': query,
|
||||
}
|
||||
return self._request('/shodan/host/search/tokens', query_args)
|
||||
|
||||
def services(self):
|
||||
"""Get a list of services that Shodan crawls
|
||||
|
||||
:returns: A dictionary containing the ports/ services that Shodan crawls for. The key is the port number and the value is the name of the service.
|
||||
"""
|
||||
return self._request('/shodan/services', {})
|
||||
|
||||
def queries(self, page=1, sort='timestamp', order='desc'):
|
||||
"""List the search queries that have been shared by other users.
|
||||
|
||||
:param page: Page number to iterate over results; each page contains 10 items
|
||||
:type page: int
|
||||
:param sort: Sort the list based on a property. Possible values are: votes, timestamp
|
||||
:type sort: str
|
||||
:param order: Whether to sort the list in ascending or descending order. Possible values are: asc, desc
|
||||
:type order: str
|
||||
|
||||
:returns: A list of saved search queries (dictionaries).
|
||||
"""
|
||||
args = {
|
||||
'page': page,
|
||||
'sort': sort,
|
||||
'order': order,
|
||||
}
|
||||
return self._request('/shodan/query', args)
|
||||
|
||||
def queries_search(self, query, page=1):
|
||||
"""Search the directory of saved search queries in Shodan.
|
||||
|
||||
:param query: The search string to look for in the search query
|
||||
:type query: str
|
||||
:param page: Page number to iterate over results; each page contains 10 items
|
||||
:type page: int
|
||||
|
||||
:returns: A list of saved search queries (dictionaries).
|
||||
"""
|
||||
args = {
|
||||
'page': page,
|
||||
'query': query,
|
||||
}
|
||||
return self._request('/shodan/query/search', args)
|
||||
|
||||
def queries_tags(self, size=10):
|
||||
"""Search the directory of saved search queries in Shodan.
|
||||
|
||||
:param query: The number of tags to return
|
||||
:type page: int
|
||||
|
||||
:returns: A list of tags.
|
||||
"""
|
||||
args = {
|
||||
'size': size,
|
||||
}
|
||||
return self._request('/shodan/query/tags', args)
|
||||
|
||||
def create_alert(self, name, ip, expires=0):
|
||||
"""Search the directory of saved search queries in Shodan.
|
||||
|
||||
:param query: The number of tags to return
|
||||
:type page: int
|
||||
|
||||
:returns: A list of tags.
|
||||
"""
|
||||
data = {
|
||||
'name': name,
|
||||
'filters': {
|
||||
'ip': ip,
|
||||
},
|
||||
'expires': expires,
|
||||
}
|
||||
|
||||
response = api_request(self.api_key, '/shodan/alert', data=data, params={}, method='post')
|
||||
|
||||
return response
|
||||
|
||||
def alerts(self, aid=None, include_expired=True):
|
||||
"""List all of the active alerts that the user created."""
|
||||
if aid:
|
||||
func = '/shodan/alert/%s/info' % aid
|
||||
else:
|
||||
func = '/shodan/alert/info'
|
||||
|
||||
response = api_request(self.api_key, func, params={
|
||||
'include_expired': include_expired,
|
||||
})
|
||||
|
||||
return response
|
||||
|
||||
def delete_alert(self, aid):
|
||||
"""Delete the alert with the given ID."""
|
||||
func = '/shodan/alert/%s' % aid
|
||||
|
||||
response = api_request(self.api_key, func, params={}, method='delete')
|
||||
|
||||
return response
|
||||
|
||||
@@ -1,20 +0,0 @@
|
||||
class WebAPIError(Exception):
|
||||
def __init__(self, value):
|
||||
self.value = value
|
||||
|
||||
def __str__(self):
|
||||
return self.value
|
||||
|
||||
|
||||
class APIError(Exception):
|
||||
"""This exception gets raised whenever a non-200 status code was returned by the Shodan API."""
|
||||
|
||||
def __init__(self, value):
|
||||
self.value = value
|
||||
|
||||
def __str__(self):
|
||||
return self.value
|
||||
|
||||
|
||||
class APITimeout(APIError):
|
||||
pass
|
||||
@@ -1,167 +0,0 @@
|
||||
import gzip
|
||||
import requests
|
||||
import json
|
||||
|
||||
from .exception import APIError
|
||||
|
||||
try:
|
||||
str
|
||||
except NameError:
|
||||
basestring = str
|
||||
|
||||
|
||||
def create_facet_string(facets):
|
||||
"""Converts a Python list of facets into a comma-separated string that can be understood by
|
||||
the Shodan API.
|
||||
"""
|
||||
facet_str = ''
|
||||
for facet in facets:
|
||||
if isinstance(facet, str):
|
||||
facet_str += facet
|
||||
else:
|
||||
facet_str += '%s:%s' % (facet[0], facet[1])
|
||||
facet_str += ','
|
||||
return facet_str[:-1]
|
||||
|
||||
|
||||
def api_request(key, function, params=None, data=None, base_url='https://api.shodan.io', method='get', retries=1):
|
||||
"""General-purpose function to create web requests to SHODAN.
|
||||
|
||||
Arguments:
|
||||
function -- name of the function you want to execute
|
||||
params -- dictionary of parameters for the function
|
||||
|
||||
Returns
|
||||
A dictionary containing the function's results.
|
||||
|
||||
"""
|
||||
# Add the API key parameter automatically
|
||||
params['key'] = key
|
||||
|
||||
# Send the request
|
||||
tries = 0
|
||||
error = False
|
||||
while tries <= retries:
|
||||
try:
|
||||
if method.lower() == 'post':
|
||||
data = requests.post(base_url + function, json.dumps(data), params=params, headers={'content-type': 'application/json'})
|
||||
elif method.lower() == 'delete':
|
||||
data = requests.delete(base_url + function, params=params)
|
||||
else:
|
||||
data = requests.get(base_url + function, params=params)
|
||||
|
||||
# Exit out of the loop
|
||||
break
|
||||
except:
|
||||
error = True
|
||||
tries += 1
|
||||
|
||||
if error and tries >= retries:
|
||||
raise APIError('Unable to connect to Shodan')
|
||||
|
||||
# Check that the API key wasn't rejected
|
||||
if data.status_code == 401:
|
||||
try:
|
||||
raise APIError(data.json()['error'])
|
||||
except:
|
||||
pass
|
||||
raise APIError('Invalid API key')
|
||||
|
||||
# Parse the text into JSON
|
||||
try:
|
||||
data = data.json()
|
||||
except:
|
||||
raise APIError('Unable to parse JSON response')
|
||||
|
||||
# Raise an exception if an error occurred
|
||||
if type(data) == dict and data.get('error', None):
|
||||
raise APIError(data['error'])
|
||||
|
||||
# Return the data
|
||||
return data
|
||||
|
||||
|
||||
def iterate_files(files, fast=False):
|
||||
"""Loop over all the records of the provided Shodan output file(s)."""
|
||||
from json import loads
|
||||
if fast:
|
||||
# Try to use ujson for parsing JSON if it's available and the user requested faster throughput
|
||||
# It's significantly faster at encoding/ decoding JSON but it doesn't support as
|
||||
# many options as the standard library. As such, we're mostly interested in using it for
|
||||
# decoding since reading/ parsing files will use up the most time.
|
||||
try:
|
||||
from ujson import loads
|
||||
except:
|
||||
pass
|
||||
|
||||
if isinstance(files, str):
|
||||
files = [files]
|
||||
|
||||
for filename in files:
|
||||
# Create a file handle depending on the filetype
|
||||
if filename.endswith('.gz'):
|
||||
fin = gzip.open(filename, 'r')
|
||||
else:
|
||||
fin = open(filename, 'r')
|
||||
|
||||
for line in fin:
|
||||
# Ensure the line has been decoded into a string to prevent errors w/ Python3
|
||||
line = line.decode('utf-8')
|
||||
|
||||
# Convert the JSON into a native Python object
|
||||
banner = loads(line)
|
||||
yield banner
|
||||
|
||||
def get_screenshot(banner):
|
||||
if 'opts' in banner and 'screenshot' in banner['opts']:
|
||||
return banner['opts']['screenshot']
|
||||
return None
|
||||
|
||||
|
||||
def get_ip(banner):
|
||||
if 'ipv6' in banner:
|
||||
return banner['ipv6']
|
||||
return banner['ip_str']
|
||||
|
||||
|
||||
def open_file(filename, mode='a', compresslevel=9):
|
||||
return gzip.open(filename, mode, compresslevel)
|
||||
|
||||
|
||||
def write_banner(fout, banner):
|
||||
line = json.dumps(banner) + '\n'
|
||||
fout.write(line.encode('utf-8'))
|
||||
|
||||
|
||||
def humanize_bytes(bytes, precision=1):
|
||||
"""Return a humanized string representation of a number of bytes.
|
||||
>>> humanize_bytes(1)
|
||||
'1 byte'
|
||||
>>> humanize_bytes(1024)
|
||||
'1.0 kB'
|
||||
>>> humanize_bytes(1024*123)
|
||||
'123.0 kB'
|
||||
>>> humanize_bytes(1024*12342)
|
||||
'12.1 MB'
|
||||
>>> humanize_bytes(1024*12342,2)
|
||||
'12.05 MB'
|
||||
>>> humanize_bytes(1024*1234,2)
|
||||
'1.21 MB'
|
||||
>>> humanize_bytes(1024*1234*1111,2)
|
||||
'1.31 GB'
|
||||
>>> humanize_bytes(1024*1234*1111,1)
|
||||
'1.3 GB'
|
||||
"""
|
||||
|
||||
if bytes == 1:
|
||||
return '1 byte'
|
||||
if bytes < 1024:
|
||||
return '%.*f %s' % (precision, bytes, "bytes")
|
||||
|
||||
suffixes = ['KB', 'MB', 'GB', 'TB', 'PB']
|
||||
multiple = 1024.0 #.0 force float on python 2
|
||||
for suffix in suffixes:
|
||||
bytes /= multiple
|
||||
if bytes < multiple:
|
||||
return '%.*f %s' % (precision, bytes, suffix)
|
||||
return '%.*f %s' % (precision, bytes, suffix)
|
||||
@@ -1,123 +0,0 @@
|
||||
import requests
|
||||
import json
|
||||
import ssl
|
||||
|
||||
from .exception import APIError
|
||||
|
||||
|
||||
class Stream:
|
||||
|
||||
base_url = 'https://stream.shodan.io'
|
||||
|
||||
def __init__(self, api_key):
|
||||
self.api_key = api_key
|
||||
|
||||
def _create_stream(self, name, timeout=None):
|
||||
# The user doesn't want to use a timeout
|
||||
# If the timeout is specified as 0 then we also don't want to have a timeout
|
||||
if ( timeout and timeout <= 0 ) or ( timeout == 0 ):
|
||||
timeout = None
|
||||
|
||||
try:
|
||||
while True:
|
||||
req = requests.get(self.base_url + name, params={'key': self.api_key}, stream=True, timeout=timeout)
|
||||
|
||||
# Status code 524 is special to Cloudflare
|
||||
# It means that no data was sent from the streaming servers which caused Cloudflare
|
||||
# to terminate the connection.
|
||||
#
|
||||
# We only want to exit if there was a timeout specified or the HTTP status code is
|
||||
# not specific to Cloudflare.
|
||||
if req.status_code != 524 or timeout >= 0:
|
||||
break
|
||||
except Exception as e:
|
||||
raise APIError('Unable to contact the Shodan Streaming API')
|
||||
|
||||
if req.status_code != 200:
|
||||
try:
|
||||
data = json.loads(req.text)
|
||||
raise APIError(data['error'])
|
||||
except APIError as e:
|
||||
raise
|
||||
except Exception as e:
|
||||
pass
|
||||
raise APIError('Invalid API key or you do not have access to the Streaming API')
|
||||
if req.encoding is None:
|
||||
req.encoding = 'utf-8'
|
||||
return req
|
||||
|
||||
def _iter_stream(self, stream, raw, timeout=None):
|
||||
for line in stream.iter_lines(decode_unicode=True):
|
||||
# The Streaming API sends out heartbeat messages that are newlines
|
||||
# We want to ignore those messages since they don't contain any data
|
||||
if line:
|
||||
if raw:
|
||||
yield line
|
||||
else:
|
||||
yield json.loads(line)
|
||||
else:
|
||||
# If the user specified a timeout then we want to keep track of how long we've
|
||||
# been getting heartbeat messages and exit the loop if it's been too long since
|
||||
# we've seen any activity.
|
||||
if timeout:
|
||||
# TODO: This is a placeholder for now but since the Streaming API added heartbeats it broke
|
||||
# the ability to use inactivity timeouts (the connection timeout still works). The timeout is
|
||||
# mostly needed when doing on-demand scans and wanting to temporarily consume data from a
|
||||
# network alert.
|
||||
pass
|
||||
|
||||
def alert(self, aid=None, timeout=None, raw=False):
|
||||
if aid:
|
||||
stream = self._create_stream('/shodan/alert/%s' % aid, timeout=timeout)
|
||||
else:
|
||||
stream = self._create_stream('/shodan/alert', timeout=timeout)
|
||||
|
||||
try:
|
||||
for line in self._iter_stream(stream, raw):
|
||||
yield line
|
||||
except requests.exceptions.ConnectionError as e:
|
||||
raise APIError('Stream timed out')
|
||||
except ssl.SSLError as e:
|
||||
raise APIError('Stream timed out')
|
||||
|
||||
def asn(self, asn, raw=False, timeout=None):
|
||||
"""
|
||||
A filtered version of the "banners" stream to only return banners that match the ASNs of interest.
|
||||
|
||||
:param asn: A list of ASN to return banner data on.
|
||||
:type asn: string[]
|
||||
"""
|
||||
stream = self._create_stream('/shodan/asn/%s' % ','.join(asn), timeout=timeout)
|
||||
for line in self._iter_stream(stream, raw):
|
||||
yield line
|
||||
|
||||
def banners(self, raw=False, timeout=None):
|
||||
"""A real-time feed of the data that Shodan is currently collecting. Note that this is only available to
|
||||
API subscription plans and for those it only returns a fraction of the data.
|
||||
"""
|
||||
stream = self._create_stream('/shodan/banners', timeout=timeout)
|
||||
for line in self._iter_stream(stream, raw):
|
||||
yield line
|
||||
|
||||
def countries(self, countries, raw=False, timeout=None):
|
||||
"""
|
||||
A filtered version of the "banners" stream to only return banners that match the countries of interest.
|
||||
|
||||
:param countries: A list of countries to return banner data on.
|
||||
:type countries: string[]
|
||||
"""
|
||||
stream = self._create_stream('/shodan/countries/%s' % ','.join(countries), timeout=timeout)
|
||||
for line in self._iter_stream(stream, raw):
|
||||
yield line
|
||||
|
||||
def ports(self, ports, raw=False, timeout=None):
|
||||
"""
|
||||
A filtered version of the "banners" stream to only return banners that match the ports of interest.
|
||||
|
||||
:param ports: A list of ports to return banner data on.
|
||||
:type ports: int[]
|
||||
"""
|
||||
stream = self._create_stream('/shodan/ports/%s' % ','.join([str(port) for port in ports]), timeout=timeout)
|
||||
for line in self._iter_stream(stream, raw):
|
||||
yield line
|
||||
|
||||
@@ -1,66 +0,0 @@
|
||||
import requests
|
||||
import json
|
||||
|
||||
from .exception import APIError
|
||||
|
||||
|
||||
class Threatnet:
|
||||
"""Wrapper around the Threatnet REST and Streaming APIs
|
||||
|
||||
:param key: The Shodan API key that can be obtained from your account page (https://account.shodan.io)
|
||||
:type key: str
|
||||
:ivar stream: An instance of `shodan.Threatnet.Stream` that provides access to the Streaming API.
|
||||
"""
|
||||
|
||||
class Stream:
|
||||
|
||||
base_url = 'https://stream.shodan.io'
|
||||
|
||||
def __init__(self, parent):
|
||||
self.parent = parent
|
||||
|
||||
def _create_stream(self, name):
|
||||
try:
|
||||
req = requests.get(self.base_url + name, params={'key': self.parent.api_key}, stream=True)
|
||||
except:
|
||||
raise APIError('Unable to contact the Shodan Streaming API')
|
||||
|
||||
if req.status_code != 200:
|
||||
try:
|
||||
raise APIError(data.json()['error'])
|
||||
except:
|
||||
pass
|
||||
raise APIError('Invalid API key or you do not have access to the Streaming API')
|
||||
return req
|
||||
|
||||
def events(self):
|
||||
stream = self._create_stream('/threatnet/events')
|
||||
for line in stream.iter_lines():
|
||||
if line:
|
||||
banner = json.loads(line)
|
||||
yield banner
|
||||
|
||||
def backscatter(self):
|
||||
stream = self._create_stream('/threatnet/backscatter')
|
||||
for line in stream.iter_lines():
|
||||
if line:
|
||||
banner = json.loads(line)
|
||||
yield banner
|
||||
|
||||
def activity(self):
|
||||
stream = self._create_stream('/threatnet/ssh')
|
||||
for line in stream.iter_lines():
|
||||
if line:
|
||||
banner = json.loads(line)
|
||||
yield banner
|
||||
|
||||
def __init__(self, key):
|
||||
"""Initializes the API object.
|
||||
|
||||
:param key: The Shodan API key.
|
||||
:type key: str
|
||||
"""
|
||||
self.api_key = key
|
||||
self.base_url = 'https://api.shodan.io'
|
||||
self.stream = self.Stream(self)
|
||||
|
||||
+32
-13
@@ -1,24 +1,43 @@
|
||||
from discovery.shodan import Shodan
|
||||
from shodan import Shodan
|
||||
from shodan import exception
|
||||
#from discovery.shodan import Shodan
|
||||
from discovery.constants import *
|
||||
|
||||
|
||||
class search_shodan():
|
||||
class search_shodan:
|
||||
|
||||
def __init__(self, host):
|
||||
self.host = host
|
||||
def __init__(self):
|
||||
self.key = shodanAPI_key
|
||||
if self.key == "":
|
||||
raise MissingKey(True)
|
||||
self.api = Shodan(self.key)
|
||||
self.hostdatarow = []
|
||||
|
||||
def run(self):
|
||||
def search_ip(self, ip):
|
||||
try:
|
||||
result = self.api.host(self.host)
|
||||
# for service in result['data']:
|
||||
# print ("%s:%s" % (service['ip_str'], service['port']))
|
||||
# print ("%s" % (service['product']))
|
||||
# print ("%s" % (service['hostnames']))
|
||||
return result
|
||||
ipaddress = ip
|
||||
results = self.api.host(ipaddress)
|
||||
technologies = []
|
||||
servicesports = []
|
||||
for result in results['data']:
|
||||
try:
|
||||
for key in result['http']['components'].keys():
|
||||
technologies.append(key)
|
||||
except KeyError as e:
|
||||
pass
|
||||
port = str(result.get('port'))
|
||||
product = str(result.get('product'))
|
||||
servicesports.append(str(product)+':'+str(port))
|
||||
technologies = list(set(technologies))
|
||||
self.hostdatarow = [
|
||||
str(results.get('ip_str')), str(results.get('hostnames')).strip('[]\''),
|
||||
str(results.get('org')), str(servicesports).replace('\'', '').strip('[]'),
|
||||
str(technologies).replace('\'', '').strip('[]')]
|
||||
except exception.APIError:
|
||||
print(ipaddress+": Not in Shodan")
|
||||
self.hostdatarow = [ipaddress, "Not in Shodan", "Not in Shodan", "Not in Shodan", "Not in Shodan"]
|
||||
|
||||
except Exception as e:
|
||||
print("SHODAN empty reply or error in the call")
|
||||
return "error"
|
||||
print("Error occurred in the Shodan IP search module: " + str(e))
|
||||
finally:
|
||||
return self.hostdatarow
|
||||
+3
-1
@@ -1,3 +1,5 @@
|
||||
beautifulsoup4>=4.7.0
|
||||
plotly>=3.4.2
|
||||
requests>=2.21.0
|
||||
requests>=2.21.0
|
||||
texttable>=1.4.0
|
||||
shodan>=1.10.0
|
||||
+65
-42
@@ -4,6 +4,8 @@ import datetime
|
||||
import getopt
|
||||
import re
|
||||
import stash
|
||||
import datetime
|
||||
import time
|
||||
from discovery import *
|
||||
from discovery.constants import *
|
||||
from lib.core import *
|
||||
@@ -139,7 +141,7 @@ def start(argv):
|
||||
print("[-] Searching in Censys.")
|
||||
from discovery import censys
|
||||
# Import locally or won't work
|
||||
search = censys.search_censys(word,limit)
|
||||
search = censys.search_censys(word, limit)
|
||||
search.process()
|
||||
all_ip = search.get_ipaddresses()
|
||||
hosts = filter(search.get_hostnames())
|
||||
@@ -194,7 +196,8 @@ def start(argv):
|
||||
elif engineitem == "googleCSE":
|
||||
print("[-] Searching in Google Custom Search.")
|
||||
try:
|
||||
search = googleCSE.search_googleCSE(word, limit, start)
|
||||
search = googleCSE.search_googleCSE(
|
||||
word, limit, start)
|
||||
search.process()
|
||||
search.store_results()
|
||||
all_emails = filter(search.get_emails())
|
||||
@@ -211,13 +214,16 @@ def start(argv):
|
||||
pass
|
||||
|
||||
elif engineitem == "google-certificates":
|
||||
print("[-] Searching in Google Certificate transparency report.")
|
||||
search = googlecertificates.search_googlecertificates(word, limit, start)
|
||||
print(
|
||||
"[-] Searching in Google Certificate transparency report.")
|
||||
search = googlecertificates.search_googlecertificates(
|
||||
word, limit, start)
|
||||
search.process()
|
||||
hosts = filter(search.get_domains())
|
||||
all_hosts.extend(hosts)
|
||||
db = stash.stash_manager()
|
||||
db.store_all(word, all_hosts, 'host', 'google-certificates')
|
||||
db.store_all(word, all_hosts, 'host',
|
||||
'google-certificates')
|
||||
|
||||
elif engineitem == "google-profiles":
|
||||
print("[-] Searching in Google profiles.")
|
||||
@@ -237,7 +243,8 @@ def start(argv):
|
||||
from discovery import huntersearch
|
||||
# Import locally or won't work.
|
||||
try:
|
||||
search = huntersearch.search_hunter(word, limit, start)
|
||||
search = huntersearch.search_hunter(
|
||||
word, limit, start)
|
||||
search.process()
|
||||
emails = filter(search.get_emails())
|
||||
all_emails.extend(emails)
|
||||
@@ -292,7 +299,8 @@ def start(argv):
|
||||
print("[-] Searching in SecurityTrails.")
|
||||
from discovery import securitytrailssearch
|
||||
try:
|
||||
search = securitytrailssearch.search_securitytrail(word)
|
||||
search = securitytrailssearch.search_securitytrail(
|
||||
word)
|
||||
search.process()
|
||||
hosts = filter(search.get_hostnames())
|
||||
all_hosts.extend(hosts)
|
||||
@@ -316,7 +324,8 @@ def start(argv):
|
||||
hosts = filter(search.get_hostnames())
|
||||
all_hosts.extend(hosts)
|
||||
db = stash.stash_manager()
|
||||
db.store_all(word, all_hosts, 'host', 'threatcrowd')
|
||||
db.store_all(word, all_hosts,
|
||||
'host', 'threatcrowd')
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@@ -394,7 +403,7 @@ def start(argv):
|
||||
|
||||
print("[-] Searching in Censys.")
|
||||
from discovery import censys
|
||||
search = censys.search_censys(word,limit)
|
||||
search = censys.search_censys(word, limit)
|
||||
search.process()
|
||||
ips = search.get_ipaddresses()
|
||||
setips = set(ips)
|
||||
@@ -432,13 +441,16 @@ def start(argv):
|
||||
db = stash.stash_manager()
|
||||
db.store_all(word, all_hosts, 'host', 'google')
|
||||
|
||||
print("[-] Searching in Google Certificate transparency report.")
|
||||
search = googlecertificates.search_googlecertificates(word, limit, start)
|
||||
print(
|
||||
"[-] Searching in Google Certificate transparency report.")
|
||||
search = googlecertificates.search_googlecertificates(
|
||||
word, limit, start)
|
||||
search.process()
|
||||
domains = filter(search.get_domains())
|
||||
all_hosts.extend(domains)
|
||||
db = stash.stash_manager()
|
||||
db.store_all(word, all_hosts, 'host', 'google-certificates')
|
||||
db.store_all(word, all_hosts, 'host',
|
||||
'google-certificates')
|
||||
|
||||
# googleplus
|
||||
|
||||
@@ -450,7 +462,8 @@ def start(argv):
|
||||
from discovery import huntersearch
|
||||
# Import locally.
|
||||
try:
|
||||
search = huntersearch.search_hunter(word, limit, start)
|
||||
search = huntersearch.search_hunter(
|
||||
word, limit, start)
|
||||
search.process()
|
||||
emails = filter(search.get_emails())
|
||||
hosts = filter(search.get_hostnames())
|
||||
@@ -500,7 +513,8 @@ def start(argv):
|
||||
hosts = filter(search.get_hostnames())
|
||||
all_hosts.extend(hosts)
|
||||
db = stash.stash_manager()
|
||||
db.store_all(word, all_hosts, 'host', 'threatcrowd')
|
||||
db.store_all(word, all_hosts,
|
||||
'host', 'threatcrowd')
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@@ -657,7 +671,8 @@ def start(argv):
|
||||
s = '.'
|
||||
range = s.join(range)
|
||||
if not analyzed_ranges.count(range):
|
||||
print(("\033[94m[-] Performing reverse lookup in " + range + "\033[1;33;40m"))
|
||||
print(
|
||||
("\033[94m[-] Performing reverse lookup in " + range + "\033[1;33;40m"))
|
||||
a = dnssearch.dns_reverse(range, True)
|
||||
a.list()
|
||||
res = a.process()
|
||||
@@ -709,29 +724,31 @@ def start(argv):
|
||||
|
||||
# Shodan search
|
||||
shodanres = []
|
||||
shodanvisited = []
|
||||
import texttable
|
||||
tab = texttable.Texttable()
|
||||
header = ["IP address", "Hostname", "Org",
|
||||
"Services:Ports", "Technologies"]
|
||||
tab.header(header)
|
||||
tab.set_cols_align(["c", "c", "c", "c", "c"])
|
||||
tab.set_cols_valign(["m", "m", "m", "m", "m"])
|
||||
tab.set_chars(['-', '|', '+', '#'])
|
||||
tab.set_cols_width([15, 20, 15, 15, 18])
|
||||
host_ip = list(set(host_ip))
|
||||
if shodan is True:
|
||||
print("\n\n\033[1;32;40m[-] Shodan DB search (passive):\n")
|
||||
if full == []:
|
||||
print('No host to search, exiting.')
|
||||
sys.exit(1)
|
||||
for x in full:
|
||||
try:
|
||||
ip = x.split(":")[1]
|
||||
if not shodanvisited.count(ip):
|
||||
print(("\tSearching for: " + ip))
|
||||
a = shodansearch.search_shodan(ip)
|
||||
shodanvisited.append(ip)
|
||||
results = a.run()
|
||||
for res in results['data']:
|
||||
shodanres.append(
|
||||
str("%s:%s - %s - %s - %s," % (res['ip_str'], res['port'], res['os'], res['isp'])))
|
||||
except Exception as e:
|
||||
pass
|
||||
print("\n [+] Shodan results:")
|
||||
print("-------------------")
|
||||
for x in shodanres:
|
||||
print(x)
|
||||
try:
|
||||
for ip in host_ip:
|
||||
print(("\tSearching for: " + ip))
|
||||
shodan = shodansearch.search_shodan()
|
||||
rowdata = shodan.search_ip(ip)
|
||||
time.sleep(2)
|
||||
tab.add_row(rowdata)
|
||||
printedtable = tab.draw()
|
||||
print("\n [+] Shodan results:")
|
||||
print("-------------------")
|
||||
print(printedtable)
|
||||
except Exception as e:
|
||||
print("Error occurred in theHarvester - Shodan search module: " + str(e))
|
||||
else:
|
||||
pass
|
||||
|
||||
@@ -757,18 +774,22 @@ def start(argv):
|
||||
db = stash.stash_manager()
|
||||
scanboarddata = db.getscanboarddata()
|
||||
latestscanresults = db.getlatestscanresults(word)
|
||||
previousscanresults = db.getlatestscanresults(word, previousday=True)
|
||||
previousscanresults = db.getlatestscanresults(
|
||||
word, previousday=True)
|
||||
latestscanchartdata = db.latestscanchartdata(word)
|
||||
scanhistorydomain = db.getscanhistorydomain(word)
|
||||
pluginscanstatistics = db.getpluginscanstatistics()
|
||||
generator = statichtmlgenerator.htmlgenerator(word)
|
||||
HTMLcode = generator.beginhtml()
|
||||
HTMLcode += generator.generatelatestscanresults(latestscanresults)
|
||||
HTMLcode += generator.generatepreviousscanresults(previousscanresults)
|
||||
HTMLcode += generator.generatepreviousscanresults(
|
||||
previousscanresults)
|
||||
graph = reportgraph.graphgenerator(word)
|
||||
HTMLcode += graph.drawlatestscangraph(word, latestscanchartdata)
|
||||
HTMLcode += graph.drawscattergraphscanhistory(word, scanhistorydomain)
|
||||
HTMLcode += generator.generatepluginscanstatistics(pluginscanstatistics)
|
||||
HTMLcode += graph.drawscattergraphscanhistory(
|
||||
word, scanhistorydomain)
|
||||
HTMLcode += generator.generatepluginscanstatistics(
|
||||
pluginscanstatistics)
|
||||
HTMLcode += generator.generatedashboardcode(scanboarddata)
|
||||
HTMLcode += '<p><span style="color: #000000;">Report generated on ' + str(
|
||||
datetime.datetime.now()) + '</span></p>'
|
||||
@@ -804,13 +825,15 @@ def start(argv):
|
||||
for x in full:
|
||||
x = x.split(":")
|
||||
if len(x) == 2:
|
||||
file.write('<host>' + '<ip>' + x[1] + '</ip><hostname>' + x[0] + '</hostname>' + '</host>')
|
||||
file.write(
|
||||
'<host>' + '<ip>' + x[1] + '</ip><hostname>' + x[0] + '</hostname>' + '</host>')
|
||||
else:
|
||||
file.write('<host>' + x + '</host>')
|
||||
for x in vhost:
|
||||
x = x.split(":")
|
||||
if len(x) == 2:
|
||||
file.write('<vhost>' + '<ip>' + x[1] + '</ip><hostname>' + x[0] + '</hostname>' + '</vhost>')
|
||||
file.write(
|
||||
'<vhost>' + '<ip>' + x[1] + '</ip><hostname>' + x[0] + '</hostname>' + '</vhost>')
|
||||
else:
|
||||
file.write('<vhost>' + x + '</vhost>')
|
||||
if shodanres != []:
|
||||
|
||||
Reference in New Issue
Block a user