diff --git a/Dockerfile b/Dockerfile index e483b7e9..07a9b6c4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM python:2-alpine RUN mkdir /app -RUN pip install requests beautifulsoup4 texttable +RUN pip install requests beautifulsoup4 texttable plotly shodan WORKDIR /app COPY . /app RUN chmod +x *.py diff --git a/discovery/shodan/__init__.py b/discovery/shodan/__init__.py deleted file mode 100644 index 321e4c68..00000000 --- a/discovery/shodan/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from discovery.shodan.api import WebAPI -from discovery.shodan.client import Shodan -from discovery.shodan.exception import APIError \ No newline at end of file diff --git a/discovery/shodan/alert.py b/discovery/shodan/alert.py deleted file mode 100644 index 50ae21df..00000000 --- a/discovery/shodan/alert.py +++ /dev/null @@ -1,9 +0,0 @@ -class Alert: - def __init__(self): - self.id = None - self.name = None - self.api_key = None - self.filters = None - self.credits = None - self.created = None - self.expires = None diff --git a/discovery/shodan/api.py b/discovery/shodan/api.py deleted file mode 100644 index 9707b523..00000000 --- a/discovery/shodan/api.py +++ /dev/null @@ -1,222 +0,0 @@ -try: - # Python 2 - from urllib2 import urlopen - from urllib import urlencode -except: - # Python 3 - from urllib.request import urlopen - from urllib.parse import urlencode - -#from json import loads #TODO FIX - -from .exception import WebAPIError - - -__all__ = ['WebAPI'] - - -class WebAPI: - """Wrapper around the SHODAN webservices API""" - - class Exploits: - - def __init__(self, parent): - self.parent = parent - - def search(self, query, sources=[], cve=None, osvdb=None, msb=None, bid=None): - """Search the entire Shodan Exploits archive using the same query syntax - as the website. - - Arguments: - query -- exploit search query; same syntax as website - - Optional arguments: - sources -- metasploit, cve, osvdb, exploitdb - cve -- CVE identifier (ex. 2010-0432) - osvdb -- OSVDB identifier (ex. 11666) - msb -- Microsoft Security Bulletin ID (ex. MS05-030) - bid -- Bugtraq identifier (ex. 13951) - - """ - if sources: - query += ' source:%s' % (','.join(sources)) - if cve: - query += ' cve:%s' % (str(cve).strip()) - if osvdb: - query += ' osvdb:%s' % (str(osvdb).strip()) - if msb: - query += ' msb:%s' % (str(msb).strip()) - if bid: - query += ' bid:%s' % (str(bid).strip()) - return self.parent._request('api', {'q': query}, service='exploits') - - class ExploitDb: - - def __init__(self, parent): - self.parent = parent - - def download(self, id): - """DEPRECATED - Download the exploit code from the ExploitDB archive. - - Arguments: - id -- ID of the ExploitDB entry - """ - query = '_id:%s' % id - return self.parent.search(query, sources=['exploitdb']) - - def search(self, query, **kwargs): - """Search the ExploitDB archive. - - Arguments: - query -- Search terms - - Returns: - A dictionary with 2 main items: matches (list) and total (int). - """ - return self.parent.search(query, sources=['exploitdb']) - - - class Msf: - - def __init__(self, parent): - self.parent = parent - - def download(self, id): - """Download a metasploit module given the fullname (id) of it. - - Arguments: - id -- fullname of the module (ex. auxiliary/admin/backupexec/dump) - - Returns: - A dictionary with the following fields: - filename -- Name of the file - content-type -- Mimetype - data -- File content - """ - query = '_id:%s' % id - return self.parent.search(query, sources=['metasploit']) - - def search(self, query, **kwargs): - """Search for a Metasploit module. - """ - return self.parent.search(query, sources=['metasploit']) - - def __init__(self, key): - """Initializes the API object. - - Arguments: - key -- your API key - - """ - print('WARNING: This class is deprecated, please upgrade to use "shodan.Shodan()" instead of shodan.WebAPI()') - self.api_key = key - self.base_url = 'http://www.shodanhq.com/api/' - self.base_exploits_url = 'https://exploits.shodan.io/' - self.exploits = self.Exploits(self) - self.exploitdb = self.ExploitDb(self.exploits) - self.msf = self.Msf(self.exploits) - - def _request(self, function, params, service='shodan'): - """General-purpose function to create web requests to SHODAN. - - Arguments: - function -- name of the function you want to execute - params -- dictionary of parameters for the function - - Returns - A JSON string containing the function's results. - - """ - # Add the API key parameter automatically - params['key'] = self.api_key - - # Determine the base_url based on which service we're interacting with - base_url = { - 'shodan': self.base_url, - 'exploits': self.base_exploits_url, - }.get(service, 'shodan') - - # Send the request - try: - data = urlopen(base_url + function + '?' + urlencode(params)).read().decode('utf-8') - except: - raise WebAPIError('Unable to connect to Shodan') - - # Parse the text from JSON to a dict - data = loads(data) - - # Raise an exception if an error occurred - if data.get('error', None): - raise WebAPIError(data['error']) - - # Return the data - return data - - def count(self, query): - """Returns the total number of search results for the query. - """ - return self._request('count', {'q': query}) - - def locations(self, query): - """Return a break-down of all the countries and cities that the results for - the given search are located in. - """ - return self._request('locations', {'q': query}) - - def fingerprint(self, banner): - """Determine the software based on the banner. - - Arguments: - banner - HTTP banner - - Returns: - A list of software that matched the given banner. - """ - return self._request('fingerprint', {'banner': banner}) - - def host(self, ip): - """Get all available information on an IP. - - Arguments: - ip -- IP of the computer - - Returns: - All available information SHODAN has on the given IP, - subject to API key restrictions. - - """ - return self._request('host', {'ip': ip}) - - def info(self): - """Returns information about the current API key, such as a list of add-ons - and other features that are enabled for the current user's API plan. - """ - return self._request('info', {}) - - def search(self, query, page=1, limit=None, offset=None): - """Search the SHODAN database. - - Arguments: - query -- search query; identical syntax to the website - - Optional arguments: - page -- page number of the search results - limit -- number of results to return - offset -- search offset to begin getting results from - - Returns: - A dictionary with 3 main items: matches, countries and total. - Visit the website for more detailed information. - - """ - args = { - 'q': query, - 'p': page, - } - if limit: - args['l'] = limit - if offset: - args['o'] = offset - - return self._request('search', args) diff --git a/discovery/shodan/cli/__init__.py b/discovery/shodan/cli/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/discovery/shodan/cli/converter/__init__.py b/discovery/shodan/cli/converter/__init__.py deleted file mode 100644 index 777ce9dd..00000000 --- a/discovery/shodan/cli/converter/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from .csvc import CsvConverter -from .excel import ExcelConverter -from .geojson import GeoJsonConverter -from .images import ImagesConverter -from .kml import KmlConverter \ No newline at end of file diff --git a/discovery/shodan/cli/converter/base.py b/discovery/shodan/cli/converter/base.py deleted file mode 100644 index 13076392..00000000 --- a/discovery/shodan/cli/converter/base.py +++ /dev/null @@ -1,8 +0,0 @@ - -class Converter: - - def __init__(self, fout): - self.fout = fout - - def process(self, fout): - pass diff --git a/discovery/shodan/cli/converter/csvc.py b/discovery/shodan/cli/converter/csvc.py deleted file mode 100644 index 5949253d..00000000 --- a/discovery/shodan/cli/converter/csvc.py +++ /dev/null @@ -1,87 +0,0 @@ - -from .base import Converter -from ...helpers import iterate_files - -from collections import MutableMapping -from csv import writer as csv_writer, excel - - -class CsvConverter(Converter): - - fields = [ - 'data', - 'hostnames', - 'ip', - 'ip_str', - 'ipv6', - 'org', - 'isp', - 'location.country_code', - 'location.city', - 'location.country_name', - 'location.latitude', - 'location.longitude', - 'os', - 'asn', - 'port', - 'transport', - 'product', - 'version', - - 'ssl.cipher.version', - 'ssl.cipher.bits', - 'ssl.cipher.name', - 'ssl.alpn', - 'ssl.versions', - 'ssl.cert.serial', - 'ssl.cert.fingerprint.sha1', - 'ssl.cert.fingerprint.sha256', - - 'html', - 'title', - ] - - def process(self, files): - writer = csv_writer(self.fout, dialect=excel) - - # Write the header - writer.writerow(self.fields) - - for banner in iterate_files(files): - try: - row = [] - for field in self.fields: - value = self.banner_field(banner, field) - row.append(value) - writer.writerow(row) - except: - pass - - def banner_field(self, banner, flat_field): - # The provided field is a collapsed form of the actual field - fields = flat_field.split('.') - - try: - current_obj = banner - for field in fields: - current_obj = current_obj[field] - - # Convert a list into a concatenated string - if isinstance(current_obj, list): - current_obj = ','.join([str(i) for i in current_obj]) - - return current_obj - except: - pass - - return '' - - def flatten(self, d, parent_key='', sep='.'): - items = [] - for k, v in d.items(): - new_key = parent_key + sep + k if parent_key else k - if isinstance(v, MutableMapping): - items.extend(flatten(v, new_key, sep=sep).items()) - else: - items.append((new_key, v)) - return dict(items) \ No newline at end of file diff --git a/discovery/shodan/cli/converter/excel.py b/discovery/shodan/cli/converter/excel.py deleted file mode 100644 index 0151d5a9..00000000 --- a/discovery/shodan/cli/converter/excel.py +++ /dev/null @@ -1,130 +0,0 @@ - -from .base import Converter -from ...helpers import iterate_files, get_ip - -from collections import defaultdict, MutableMapping -from xlsxwriter import Workbook - - -class ExcelConverter(Converter): - - fields = [ - 'port', - 'timestamp', - 'data', - 'hostnames', - 'org', - 'isp', - 'location.country_name', - 'location.country_code', - 'location.city', - 'os', - 'asn', - 'transport', - 'product', - 'version', - - 'http.server', - 'http.title', - ] - - field_names = { - 'org': 'Organization', - 'isp': 'ISP', - 'location.country_code': 'Country ISO Code', - 'location.country_name': 'Country', - 'location.city': 'City', - 'os': 'OS', - 'asn': 'ASN', - - 'http.server': 'Web Server', - 'http.title': 'Website Title', - } - - def process(self, files): - # Get the filename from the already-open file handle - filename = self.fout.name - - # Close the existing file as the XlsxWriter library handles that for us - self.fout.close() - - # Create the new workbook - workbook = Workbook(filename) - - # Define some common styles/ formats - bold = workbook.add_format({ - 'bold': 1, - }) - - # Create the main worksheet where all the raw data is shown - main_sheet = workbook.add_worksheet('Raw Data') - - # Write the header - main_sheet.write(0, 0, 'IP', bold) # The IP field can be either ip_str or ipv6 so we treat it differently - main_sheet.set_column(0, 0, 20) - - row = 0 - col = 1 - for field in self.fields: - name = self.field_names.get(field, field.capitalize()) - main_sheet.write(row, col, name, bold) - col += 1 - row += 1 - - total = 0 - ports = defaultdict(int) - for banner in iterate_files(files): - try: - # Build the list that contains all the relevant values - data = [] - for field in self.fields: - value = self.banner_field(banner, field) - data.append(value) - - # Write those values to the main workbook - # Starting off w/ the special "IP" property - main_sheet.write_string(row, 0, get_ip(banner)) - col = 1 - - for value in data: - main_sheet.write(row, col, value) - col += 1 - row += 1 - except: - pass - - # Aggregate summary information - total += 1 - ports[banner['port']] += 1 - - summary_sheet = workbook.add_worksheet('Summary') - summary_sheet.write(0, 0, 'Total', bold) - summary_sheet.write(0, 1, total) - - # Ports Distribution - summary_sheet.write(0, 3, 'Ports Distribution', bold) - row = 1 - col = 3 - for key, value in sorted(ports.items(), reverse=True, key=lambda kv: (kv[1], kv[0])): - summary_sheet.write(row, col, key) - summary_sheet.write(row, col + 1, value) - row += 1 - - def banner_field(self, banner, flat_field): - # The provided field is a collapsed form of the actual field - fields = flat_field.split('.') - - try: - current_obj = banner - for field in fields: - current_obj = current_obj[field] - - # Convert a list into a concatenated string - if isinstance(current_obj, list): - current_obj = ','.join([str(i) for i in current_obj]) - - return current_obj - except: - pass - - return '' \ No newline at end of file diff --git a/discovery/shodan/cli/converter/geojson.py b/discovery/shodan/cli/converter/geojson.py deleted file mode 100644 index c2ea1984..00000000 --- a/discovery/shodan/cli/converter/geojson.py +++ /dev/null @@ -1,57 +0,0 @@ - -from base import Converter -from ...helpers import get_ip, iterate_files - -class GeoJsonConverter(Converter): - - def header(self): - self.fout.write("""{ - "type": "FeatureCollection", - "features": [ - """) - - def footer(self): - self.fout.write("""{ }]}""") - - def process(self, files): - # Write the header - self.header() - - hosts = {} - for banner in iterate_files(files): - ip = get_ip(banner) - if not ip: - continue - - if ip not in hosts: - hosts[ip] = banner - hosts[ip]['ports'] = [] - - hosts[ip]['ports'].append(banner['port']) - - for ip, host in iter(hosts.items()): - self.write(host) - - self.footer() - - - def write(self, host): - try: - ip = get_ip(host) - lat, lon = host['location']['latitude'], host['location']['longitude'] - - feature = """{ - "type": "Feature", - "id": "{}", - "properties": { - "name": "{}" - }, - "geometry": { - "type": "Point", - "coordinates": [{}, {}] - } - }""".format(ip, ip, lat, lon) - - self.fout.write(feature) - except Exception as e: - pass diff --git a/discovery/shodan/cli/converter/images.py b/discovery/shodan/cli/converter/images.py deleted file mode 100644 index f164b69e..00000000 --- a/discovery/shodan/cli/converter/images.py +++ /dev/null @@ -1,51 +0,0 @@ - -from base import Converter -from ...helpers import iterate_files, get_ip, get_screenshot - -# Needed for decoding base64-strings in Python3 -from codecs import decode - -import os - - -class ImagesConverter(Converter): - - # The Images converter is special in that it creates a directory and there's - # special code in the Shodan CLI that relies on the "dirname" property to let - # the user know where the images have been stored. - dirname = None - - def process(self, files): - # Get the filename from the already-open file handle and use it as - # the directory name to store the images. - self.dirname = self.fout.name[:-7] + '-images' - - # Remove the original file that was created - self.fout.close() - os.unlink(self.fout.name) - - # Create the directory if it doesn't yet exist - if not os.path.exists(self.dirname): - os.mkdir(self.dirname) - - # Close the existing file as the XlsxWriter library handles that for us - self.fout.close() - - # Loop through all the banners in the data file - for banner in iterate_files(files): - screenshot = get_screenshot(banner) - if screenshot: - filename = '{}/{}-{}'.format(self.dirname, get_ip(banner), banner['port']) - - # If a file with the name already exists then count up until we - # create a new, unique filename - counter = 0 - tmpname = filename - while os.path.exists(tmpname + '.jpg'): - tmpname = '{}-{}'.format(filename, counter) - counter += 1 - filename = tmpname + '.jpg' - - fout = open(filename, 'wb') - fout.write(decode(screenshot['data'].encode(), 'base64')) - fout.close() diff --git a/discovery/shodan/cli/converter/kml.py b/discovery/shodan/cli/converter/kml.py deleted file mode 100644 index 107fed1b..00000000 --- a/discovery/shodan/cli/converter/kml.py +++ /dev/null @@ -1,127 +0,0 @@ - -from .base import Converter -from ...helpers import iterate_files - -class KmlConverter(Converter): - - def header(self): - self.fout.write(""" - - """) - - def footer(self): - self.fout.write("""""") - - def process(self, files): - # Write the header - self.header() - - hosts = {} - for banner in iterate_files(files): - ip = banner.get('ip_str', banner.get('ipv6', None)) - if not ip: - continue - - if ip not in hosts: - hosts[ip] = banner - hosts[ip]['ports'] = [] - - hosts[ip]['ports'].append(banner['port']) - - for ip, host in iter(hosts.items()): - self.write(host) - - self.footer() - - - def write(self, host): - try: - ip = host.get('ip_str', host.get('ipv6', None)) - lat, lon = host['location']['latitude'], host['location']['longitude'] - - placemark = '{}]]>'.format(ip) - placemark += '{0}'.format(host['hostnames'][0]) - - test = """ - - - - - - - - - - - - - - - -
CityAlbuquerque
CountryUnited States
OrganizationNexcess.net L.L.C.
-

Ports

-