From 6843b99a760b702bbab1a4acbb125a9a67c4e951 Mon Sep 17 00:00:00 2001 From: Janos Zold Date: Fri, 4 Jan 2019 16:28:11 +0000 Subject: [PATCH 1/3] Implement Shodan parser using the API --- discovery/shodan/__init__.py | 3 - discovery/shodan/alert.py | 9 - discovery/shodan/api.py | 222 --------- discovery/shodan/cli/__init__.py | 0 discovery/shodan/cli/converter/__init__.py | 5 - discovery/shodan/cli/converter/base.py | 8 - discovery/shodan/cli/converter/csvc.py | 87 ---- discovery/shodan/cli/converter/excel.py | 130 ------ discovery/shodan/cli/converter/geojson.py | 57 --- discovery/shodan/cli/converter/images.py | 51 --- discovery/shodan/cli/converter/kml.py | 127 ------ discovery/shodan/cli/helpers.py | 23 - discovery/shodan/cli/settings.py | 10 - discovery/shodan/cli/worldmap.py | 251 ---------- discovery/shodan/client.py | 508 --------------------- discovery/shodan/exception.py | 20 - discovery/shodan/helpers.py | 167 ------- discovery/shodan/stream.py | 123 ----- discovery/shodan/threatnet.py | 66 --- discovery/shodansearch.py | 45 +- requirements.txt | 4 +- theHarvester.py | 106 +++-- 22 files changed, 99 insertions(+), 1923 deletions(-) delete mode 100644 discovery/shodan/__init__.py delete mode 100644 discovery/shodan/alert.py delete mode 100644 discovery/shodan/api.py delete mode 100644 discovery/shodan/cli/__init__.py delete mode 100644 discovery/shodan/cli/converter/__init__.py delete mode 100644 discovery/shodan/cli/converter/base.py delete mode 100644 discovery/shodan/cli/converter/csvc.py delete mode 100644 discovery/shodan/cli/converter/excel.py delete mode 100644 discovery/shodan/cli/converter/geojson.py delete mode 100644 discovery/shodan/cli/converter/images.py delete mode 100644 discovery/shodan/cli/converter/kml.py delete mode 100644 discovery/shodan/cli/helpers.py delete mode 100644 discovery/shodan/cli/settings.py delete mode 100755 discovery/shodan/cli/worldmap.py delete mode 100644 discovery/shodan/client.py delete mode 100644 discovery/shodan/exception.py delete mode 100644 discovery/shodan/helpers.py delete mode 100644 discovery/shodan/stream.py delete mode 100644 discovery/shodan/threatnet.py diff --git a/discovery/shodan/__init__.py b/discovery/shodan/__init__.py deleted file mode 100644 index 321e4c68..00000000 --- a/discovery/shodan/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from discovery.shodan.api import WebAPI -from discovery.shodan.client import Shodan -from discovery.shodan.exception import APIError \ No newline at end of file diff --git a/discovery/shodan/alert.py b/discovery/shodan/alert.py deleted file mode 100644 index 50ae21df..00000000 --- a/discovery/shodan/alert.py +++ /dev/null @@ -1,9 +0,0 @@ -class Alert: - def __init__(self): - self.id = None - self.name = None - self.api_key = None - self.filters = None - self.credits = None - self.created = None - self.expires = None diff --git a/discovery/shodan/api.py b/discovery/shodan/api.py deleted file mode 100644 index 9707b523..00000000 --- a/discovery/shodan/api.py +++ /dev/null @@ -1,222 +0,0 @@ -try: - # Python 2 - from urllib2 import urlopen - from urllib import urlencode -except: - # Python 3 - from urllib.request import urlopen - from urllib.parse import urlencode - -#from json import loads #TODO FIX - -from .exception import WebAPIError - - -__all__ = ['WebAPI'] - - -class WebAPI: - """Wrapper around the SHODAN webservices API""" - - class Exploits: - - def __init__(self, parent): - self.parent = parent - - def search(self, query, sources=[], cve=None, osvdb=None, msb=None, bid=None): - """Search the entire Shodan Exploits archive using the same query syntax - as the website. - - Arguments: - query -- exploit search query; same syntax as website - - Optional arguments: - sources -- metasploit, cve, osvdb, exploitdb - cve -- CVE identifier (ex. 2010-0432) - osvdb -- OSVDB identifier (ex. 11666) - msb -- Microsoft Security Bulletin ID (ex. MS05-030) - bid -- Bugtraq identifier (ex. 13951) - - """ - if sources: - query += ' source:%s' % (','.join(sources)) - if cve: - query += ' cve:%s' % (str(cve).strip()) - if osvdb: - query += ' osvdb:%s' % (str(osvdb).strip()) - if msb: - query += ' msb:%s' % (str(msb).strip()) - if bid: - query += ' bid:%s' % (str(bid).strip()) - return self.parent._request('api', {'q': query}, service='exploits') - - class ExploitDb: - - def __init__(self, parent): - self.parent = parent - - def download(self, id): - """DEPRECATED - Download the exploit code from the ExploitDB archive. - - Arguments: - id -- ID of the ExploitDB entry - """ - query = '_id:%s' % id - return self.parent.search(query, sources=['exploitdb']) - - def search(self, query, **kwargs): - """Search the ExploitDB archive. - - Arguments: - query -- Search terms - - Returns: - A dictionary with 2 main items: matches (list) and total (int). - """ - return self.parent.search(query, sources=['exploitdb']) - - - class Msf: - - def __init__(self, parent): - self.parent = parent - - def download(self, id): - """Download a metasploit module given the fullname (id) of it. - - Arguments: - id -- fullname of the module (ex. auxiliary/admin/backupexec/dump) - - Returns: - A dictionary with the following fields: - filename -- Name of the file - content-type -- Mimetype - data -- File content - """ - query = '_id:%s' % id - return self.parent.search(query, sources=['metasploit']) - - def search(self, query, **kwargs): - """Search for a Metasploit module. - """ - return self.parent.search(query, sources=['metasploit']) - - def __init__(self, key): - """Initializes the API object. - - Arguments: - key -- your API key - - """ - print('WARNING: This class is deprecated, please upgrade to use "shodan.Shodan()" instead of shodan.WebAPI()') - self.api_key = key - self.base_url = 'http://www.shodanhq.com/api/' - self.base_exploits_url = 'https://exploits.shodan.io/' - self.exploits = self.Exploits(self) - self.exploitdb = self.ExploitDb(self.exploits) - self.msf = self.Msf(self.exploits) - - def _request(self, function, params, service='shodan'): - """General-purpose function to create web requests to SHODAN. - - Arguments: - function -- name of the function you want to execute - params -- dictionary of parameters for the function - - Returns - A JSON string containing the function's results. - - """ - # Add the API key parameter automatically - params['key'] = self.api_key - - # Determine the base_url based on which service we're interacting with - base_url = { - 'shodan': self.base_url, - 'exploits': self.base_exploits_url, - }.get(service, 'shodan') - - # Send the request - try: - data = urlopen(base_url + function + '?' + urlencode(params)).read().decode('utf-8') - except: - raise WebAPIError('Unable to connect to Shodan') - - # Parse the text from JSON to a dict - data = loads(data) - - # Raise an exception if an error occurred - if data.get('error', None): - raise WebAPIError(data['error']) - - # Return the data - return data - - def count(self, query): - """Returns the total number of search results for the query. - """ - return self._request('count', {'q': query}) - - def locations(self, query): - """Return a break-down of all the countries and cities that the results for - the given search are located in. - """ - return self._request('locations', {'q': query}) - - def fingerprint(self, banner): - """Determine the software based on the banner. - - Arguments: - banner - HTTP banner - - Returns: - A list of software that matched the given banner. - """ - return self._request('fingerprint', {'banner': banner}) - - def host(self, ip): - """Get all available information on an IP. - - Arguments: - ip -- IP of the computer - - Returns: - All available information SHODAN has on the given IP, - subject to API key restrictions. - - """ - return self._request('host', {'ip': ip}) - - def info(self): - """Returns information about the current API key, such as a list of add-ons - and other features that are enabled for the current user's API plan. - """ - return self._request('info', {}) - - def search(self, query, page=1, limit=None, offset=None): - """Search the SHODAN database. - - Arguments: - query -- search query; identical syntax to the website - - Optional arguments: - page -- page number of the search results - limit -- number of results to return - offset -- search offset to begin getting results from - - Returns: - A dictionary with 3 main items: matches, countries and total. - Visit the website for more detailed information. - - """ - args = { - 'q': query, - 'p': page, - } - if limit: - args['l'] = limit - if offset: - args['o'] = offset - - return self._request('search', args) diff --git a/discovery/shodan/cli/__init__.py b/discovery/shodan/cli/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/discovery/shodan/cli/converter/__init__.py b/discovery/shodan/cli/converter/__init__.py deleted file mode 100644 index 777ce9dd..00000000 --- a/discovery/shodan/cli/converter/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from .csvc import CsvConverter -from .excel import ExcelConverter -from .geojson import GeoJsonConverter -from .images import ImagesConverter -from .kml import KmlConverter \ No newline at end of file diff --git a/discovery/shodan/cli/converter/base.py b/discovery/shodan/cli/converter/base.py deleted file mode 100644 index 13076392..00000000 --- a/discovery/shodan/cli/converter/base.py +++ /dev/null @@ -1,8 +0,0 @@ - -class Converter: - - def __init__(self, fout): - self.fout = fout - - def process(self, fout): - pass diff --git a/discovery/shodan/cli/converter/csvc.py b/discovery/shodan/cli/converter/csvc.py deleted file mode 100644 index 5949253d..00000000 --- a/discovery/shodan/cli/converter/csvc.py +++ /dev/null @@ -1,87 +0,0 @@ - -from .base import Converter -from ...helpers import iterate_files - -from collections import MutableMapping -from csv import writer as csv_writer, excel - - -class CsvConverter(Converter): - - fields = [ - 'data', - 'hostnames', - 'ip', - 'ip_str', - 'ipv6', - 'org', - 'isp', - 'location.country_code', - 'location.city', - 'location.country_name', - 'location.latitude', - 'location.longitude', - 'os', - 'asn', - 'port', - 'transport', - 'product', - 'version', - - 'ssl.cipher.version', - 'ssl.cipher.bits', - 'ssl.cipher.name', - 'ssl.alpn', - 'ssl.versions', - 'ssl.cert.serial', - 'ssl.cert.fingerprint.sha1', - 'ssl.cert.fingerprint.sha256', - - 'html', - 'title', - ] - - def process(self, files): - writer = csv_writer(self.fout, dialect=excel) - - # Write the header - writer.writerow(self.fields) - - for banner in iterate_files(files): - try: - row = [] - for field in self.fields: - value = self.banner_field(banner, field) - row.append(value) - writer.writerow(row) - except: - pass - - def banner_field(self, banner, flat_field): - # The provided field is a collapsed form of the actual field - fields = flat_field.split('.') - - try: - current_obj = banner - for field in fields: - current_obj = current_obj[field] - - # Convert a list into a concatenated string - if isinstance(current_obj, list): - current_obj = ','.join([str(i) for i in current_obj]) - - return current_obj - except: - pass - - return '' - - def flatten(self, d, parent_key='', sep='.'): - items = [] - for k, v in d.items(): - new_key = parent_key + sep + k if parent_key else k - if isinstance(v, MutableMapping): - items.extend(flatten(v, new_key, sep=sep).items()) - else: - items.append((new_key, v)) - return dict(items) \ No newline at end of file diff --git a/discovery/shodan/cli/converter/excel.py b/discovery/shodan/cli/converter/excel.py deleted file mode 100644 index 0151d5a9..00000000 --- a/discovery/shodan/cli/converter/excel.py +++ /dev/null @@ -1,130 +0,0 @@ - -from .base import Converter -from ...helpers import iterate_files, get_ip - -from collections import defaultdict, MutableMapping -from xlsxwriter import Workbook - - -class ExcelConverter(Converter): - - fields = [ - 'port', - 'timestamp', - 'data', - 'hostnames', - 'org', - 'isp', - 'location.country_name', - 'location.country_code', - 'location.city', - 'os', - 'asn', - 'transport', - 'product', - 'version', - - 'http.server', - 'http.title', - ] - - field_names = { - 'org': 'Organization', - 'isp': 'ISP', - 'location.country_code': 'Country ISO Code', - 'location.country_name': 'Country', - 'location.city': 'City', - 'os': 'OS', - 'asn': 'ASN', - - 'http.server': 'Web Server', - 'http.title': 'Website Title', - } - - def process(self, files): - # Get the filename from the already-open file handle - filename = self.fout.name - - # Close the existing file as the XlsxWriter library handles that for us - self.fout.close() - - # Create the new workbook - workbook = Workbook(filename) - - # Define some common styles/ formats - bold = workbook.add_format({ - 'bold': 1, - }) - - # Create the main worksheet where all the raw data is shown - main_sheet = workbook.add_worksheet('Raw Data') - - # Write the header - main_sheet.write(0, 0, 'IP', bold) # The IP field can be either ip_str or ipv6 so we treat it differently - main_sheet.set_column(0, 0, 20) - - row = 0 - col = 1 - for field in self.fields: - name = self.field_names.get(field, field.capitalize()) - main_sheet.write(row, col, name, bold) - col += 1 - row += 1 - - total = 0 - ports = defaultdict(int) - for banner in iterate_files(files): - try: - # Build the list that contains all the relevant values - data = [] - for field in self.fields: - value = self.banner_field(banner, field) - data.append(value) - - # Write those values to the main workbook - # Starting off w/ the special "IP" property - main_sheet.write_string(row, 0, get_ip(banner)) - col = 1 - - for value in data: - main_sheet.write(row, col, value) - col += 1 - row += 1 - except: - pass - - # Aggregate summary information - total += 1 - ports[banner['port']] += 1 - - summary_sheet = workbook.add_worksheet('Summary') - summary_sheet.write(0, 0, 'Total', bold) - summary_sheet.write(0, 1, total) - - # Ports Distribution - summary_sheet.write(0, 3, 'Ports Distribution', bold) - row = 1 - col = 3 - for key, value in sorted(ports.items(), reverse=True, key=lambda kv: (kv[1], kv[0])): - summary_sheet.write(row, col, key) - summary_sheet.write(row, col + 1, value) - row += 1 - - def banner_field(self, banner, flat_field): - # The provided field is a collapsed form of the actual field - fields = flat_field.split('.') - - try: - current_obj = banner - for field in fields: - current_obj = current_obj[field] - - # Convert a list into a concatenated string - if isinstance(current_obj, list): - current_obj = ','.join([str(i) for i in current_obj]) - - return current_obj - except: - pass - - return '' \ No newline at end of file diff --git a/discovery/shodan/cli/converter/geojson.py b/discovery/shodan/cli/converter/geojson.py deleted file mode 100644 index c2ea1984..00000000 --- a/discovery/shodan/cli/converter/geojson.py +++ /dev/null @@ -1,57 +0,0 @@ - -from base import Converter -from ...helpers import get_ip, iterate_files - -class GeoJsonConverter(Converter): - - def header(self): - self.fout.write("""{ - "type": "FeatureCollection", - "features": [ - """) - - def footer(self): - self.fout.write("""{ }]}""") - - def process(self, files): - # Write the header - self.header() - - hosts = {} - for banner in iterate_files(files): - ip = get_ip(banner) - if not ip: - continue - - if ip not in hosts: - hosts[ip] = banner - hosts[ip]['ports'] = [] - - hosts[ip]['ports'].append(banner['port']) - - for ip, host in iter(hosts.items()): - self.write(host) - - self.footer() - - - def write(self, host): - try: - ip = get_ip(host) - lat, lon = host['location']['latitude'], host['location']['longitude'] - - feature = """{ - "type": "Feature", - "id": "{}", - "properties": { - "name": "{}" - }, - "geometry": { - "type": "Point", - "coordinates": [{}, {}] - } - }""".format(ip, ip, lat, lon) - - self.fout.write(feature) - except Exception as e: - pass diff --git a/discovery/shodan/cli/converter/images.py b/discovery/shodan/cli/converter/images.py deleted file mode 100644 index f164b69e..00000000 --- a/discovery/shodan/cli/converter/images.py +++ /dev/null @@ -1,51 +0,0 @@ - -from base import Converter -from ...helpers import iterate_files, get_ip, get_screenshot - -# Needed for decoding base64-strings in Python3 -from codecs import decode - -import os - - -class ImagesConverter(Converter): - - # The Images converter is special in that it creates a directory and there's - # special code in the Shodan CLI that relies on the "dirname" property to let - # the user know where the images have been stored. - dirname = None - - def process(self, files): - # Get the filename from the already-open file handle and use it as - # the directory name to store the images. - self.dirname = self.fout.name[:-7] + '-images' - - # Remove the original file that was created - self.fout.close() - os.unlink(self.fout.name) - - # Create the directory if it doesn't yet exist - if not os.path.exists(self.dirname): - os.mkdir(self.dirname) - - # Close the existing file as the XlsxWriter library handles that for us - self.fout.close() - - # Loop through all the banners in the data file - for banner in iterate_files(files): - screenshot = get_screenshot(banner) - if screenshot: - filename = '{}/{}-{}'.format(self.dirname, get_ip(banner), banner['port']) - - # If a file with the name already exists then count up until we - # create a new, unique filename - counter = 0 - tmpname = filename - while os.path.exists(tmpname + '.jpg'): - tmpname = '{}-{}'.format(filename, counter) - counter += 1 - filename = tmpname + '.jpg' - - fout = open(filename, 'wb') - fout.write(decode(screenshot['data'].encode(), 'base64')) - fout.close() diff --git a/discovery/shodan/cli/converter/kml.py b/discovery/shodan/cli/converter/kml.py deleted file mode 100644 index 107fed1b..00000000 --- a/discovery/shodan/cli/converter/kml.py +++ /dev/null @@ -1,127 +0,0 @@ - -from .base import Converter -from ...helpers import iterate_files - -class KmlConverter(Converter): - - def header(self): - self.fout.write(""" - - """) - - def footer(self): - self.fout.write("""""") - - def process(self, files): - # Write the header - self.header() - - hosts = {} - for banner in iterate_files(files): - ip = banner.get('ip_str', banner.get('ipv6', None)) - if not ip: - continue - - if ip not in hosts: - hosts[ip] = banner - hosts[ip]['ports'] = [] - - hosts[ip]['ports'].append(banner['port']) - - for ip, host in iter(hosts.items()): - self.write(host) - - self.footer() - - - def write(self, host): - try: - ip = host.get('ip_str', host.get('ipv6', None)) - lat, lon = host['location']['latitude'], host['location']['longitude'] - - placemark = '{}]]>'.format(ip) - placemark += '{0}'.format(host['hostnames'][0]) - - test = """ - - - - - - - - - - - - - - - -
CityAlbuquerque
CountryUnited States
OrganizationNexcess.net L.L.C.
-

Ports

-
    - """ - - placemark += '

    Ports

      ' - - for port in host['ports']: - placemark += """ -
    • {} -
      • - """.format(port) - - placemark += '
    ' - - placemark += """ -
    View Details
    -
    powered by Shodan
    - """.format(ip) - - placemark += ']]>' - placemark += '{},{}'.format(lon, lat) - placemark += '' - - self.fout.write(placemark.encode('utf-8')) - except Exception as e: - pass diff --git a/discovery/shodan/cli/helpers.py b/discovery/shodan/cli/helpers.py deleted file mode 100644 index 21a4893c..00000000 --- a/discovery/shodan/cli/helpers.py +++ /dev/null @@ -1,23 +0,0 @@ -''' -Helper methods to create your own CLI commands. -''' -import click -import os - -from .settings import SHODAN_CONFIG_DIR - -def get_api_key(): - '''Returns the API key of the current logged-in user.''' - shodan_dir = os.path.expanduser(SHODAN_CONFIG_DIR) - keyfile = shodan_dir + '/api_key' - - # If the file doesn't yet exist let the user know that they need to - # initialize the shodan cli - if not os.path.exists(keyfile): - raise click.ClickException('Please run "shodan init " before using this command') - - # Make sure it is a read-only file - os.chmod(keyfile, 0o600) - - with open(keyfile, 'r') as fin: - return fin.read().strip() diff --git a/discovery/shodan/cli/settings.py b/discovery/shodan/cli/settings.py deleted file mode 100644 index a41e0189..00000000 --- a/discovery/shodan/cli/settings.py +++ /dev/null @@ -1,10 +0,0 @@ - -SHODAN_CONFIG_DIR = '~/.shodan/' -COLORIZE_FIELDS = { - 'ip_str': 'green', - 'port': 'yellow', - 'data': 'white', - 'hostnames': 'magenta', - 'org': 'cyan', - 'vulns': 'red', -} diff --git a/discovery/shodan/cli/worldmap.py b/discovery/shodan/cli/worldmap.py deleted file mode 100755 index a8a8e6ee..00000000 --- a/discovery/shodan/cli/worldmap.py +++ /dev/null @@ -1,251 +0,0 @@ -#!/usr/bin/env python -''' -F-Secure Virus World Map console edition - -See README.md for more details - -Copyright 2012-2013 Jyrki Muukkonen - -Released under the MIT license. -See LICENSE.txt or http://www.opensource.org/licenses/mit-license.php - -ASCII map in map-world-01.txt is copyright: - "Map 1998 Matthew Thomas. Freely usable as long as this line is included" - -''' -import curses -import locale -import random -import time - -from discovery.shodan.helpers import get_ip - - -MAPS = { - 'world': { - # offset (as (y, x) for curses...) - 'corners': (1, 4, 23, 73), - # lat top, lon left, lat bottom, lon right - 'coords': [90.0, -180.0, -90.0, 180.0], - 'data': ''' - . _..::__: ,-"-"._ |7 , _,.__ - _.___ _ _<_>`!(._`.`-. / _._ `_ ,_/ ' '-._.---.-.__ - .{ " " `-==,',._\{ \ / {) / _ ">_,-' ` mt-2_ - \_.:--. `._ )`^-. "' , [_/( __,/-' - '"' \ " _L oD_,--' ) /. (| - | ,' _)_.\\._<> 6 _,' / ' - `. / [_/_'` `"( <'} ) - \\ .-. ) / `-'"..' `:._ _) ' - ` \ ( `( / `:\ > \ ,-^. /' ' - `._, "" | \`' \| ?_) {\ - `=.---. `._._ ,' "` |' ,- '. - | `-._ | / `:`<_|h--._ - ( > . | , `=.__.`-'\ - `. / | |{| ,-.,\ . - | ,' \ / `' ," \ - | / |_' | __ / - | | '-' `-' \. - |/ " / - \. ' - - ,/ ______._.--._ _..---.---------._ - ,-----"-..?----_/ ) _,-'" " ( - Map 1998 Matthew Thomas. Freely usable as long as this line is included -''' - } -} - - -class AsciiMap(object): - """ - Helper class for handling map drawing and coordinate calculations - """ - def __init__(self, map_name='world', map_conf=None, window=None, encoding=None): - if map_conf is None: - map_conf = MAPS[map_name] - self.map = map_conf['data'] - self.coords = map_conf['coords'] - self.corners = map_conf['corners'] - if window is None: - window = curses.newwin(0, 0) - self.window = window - - self.data = [] - self.data_timestamp = None - - # JSON contents _should_ be UTF8 (so, python internal unicode here...) - if encoding is None: - encoding = locale.getpreferredencoding() - self.encoding = encoding - - # check if we can use transparent background or not - if curses.can_change_color(): - curses.use_default_colors() - background = -1 - else: - background = curses.COLOR_BLACK - - tmp_colors = [ - ('red', curses.COLOR_RED, background), - ('blue', curses.COLOR_BLUE, background), - ('pink', curses.COLOR_MAGENTA, background) - ] - - self.colors = {} - if curses.has_colors(): - for i, (name, fgcolor, bgcolor) in enumerate(tmp_colors, 1): - curses.init_pair(i, fgcolor, bgcolor) - self.colors[name] = i - - def latlon_to_coords(self, lat, lon): - """ - Convert lat/lon coordinates to character positions. - Very naive version, assumes that we are drawing the whole world - TODO: filter out stuff that doesn't fit - TODO: make it possible to use "zoomed" maps - """ - width = (self.corners[3]-self.corners[1]) - height = (self.corners[2]-self.corners[0]) - - # change to 0-180, 0-360 - abs_lat = -lat+90 - abs_lon = lon+180 - x = (abs_lon/360.0)*width + self.corners[1] - y = (abs_lat/180.0)*height + self.corners[0] - return int(x), int(y) - - def set_data(self, data): - """ - Set / convert internal data. - For now it just selects a random set to show. - """ - entries = [] - - # Grab 5 random banners to display - for banner in random.sample(data, min(len(data), 5)): - desc = '{} -> {} / {}'.format(get_ip(banner), banner['port'], banner['location']['country_code']) - if banner['location']['city']: - desc += ' {}'.format(banner['location']['city']) - - if 'tags' in banner and banner['tags']: - desc += ' / {}'.format(','.join(banner['tags'])) - - entry = ( - float(banner['location']['latitude']), - float(banner['location']['longitude']), - '*', - desc, - curses.A_BOLD, - 'red', - ) - entries.append(entry) - self.data = entries - - def draw(self, target): - """ Draw internal data to curses window """ - self.window.clear() - self.window.addstr(0, 0, self.map) - - # FIXME: position to be defined in map config? - row = self.corners[2]-6 - items_to_show = 5 - for lat, lon, char, desc, attrs, color in self.data: - # to make this work almost everywhere. see http://docs.python.org/2/library/curses.html - if desc: - desc = desc.encode(self.encoding, 'ignore') - if items_to_show <= 0: - break - char_x, char_y = self.latlon_to_coords(lat, lon) - if self.colors and color: - attrs |= curses.color_pair(self.colors[color]) - self.window.addstr(char_y, char_x, char, attrs) - if desc: - det_show = "%s %s" % (char, desc) - else: - det_show = None - - if det_show is not None: - try: - self.window.addstr(row, 1, det_show, attrs) - row += 1 - items_to_show -= 1 - except Exception: - # FIXME: check window size before addstr() - break - self.window.overwrite(target) - self.window.leaveok(1) - - -class MapApp(object): - """ Virus World Map ncurses application """ - def __init__(self, api): - self.api = api - self.data = None - self.last_fetch = 0 - self.sleep = 10 # tenths of seconds, for curses.halfdelay() - self.polling_interval = 60 - - def fetch_data(self, epoch_now, force_refresh=False): - """ (Re)fetch data from JSON stream """ - refresh = False - if force_refresh or self.data is None: - refresh = True - else: - if self.last_fetch + self.polling_interval <= epoch_now: - refresh = True - - if refresh: - try: - # Grab 20 banners from the main stream - banners = [] - for banner in self.api.stream.banners(): - if 'location' in banner and banner['location']['latitude']: - banners.append(banner) - if len(banners) >= 20: - break - self.data = banners - self.last_fetch = epoch_now - except Exception: - raise - return refresh - - def run(self, scr): - """ Initialize and run the application """ - m = AsciiMap() - curses.halfdelay(self.sleep) - while True: - now = int(time.time()) - refresh = self.fetch_data(now) - m.set_data(self.data) - m.draw(scr) - scr.addstr(0, 1, 'Shodan Radar', curses.A_BOLD) - scr.addstr(0, 40, time.strftime("%c UTC", time.gmtime(now)).rjust(37), curses.A_BOLD) - - # Key Input - # q - Quit - event = scr.getch() - if event == ord('q'): - break - - # redraw window (to fix encoding/rendering bugs and to hide other messages to same tty) - # user pressed 'r' or new data was fetched - if refresh: - m.window.redrawwin() - - -def launch_map(api): - app = MapApp(api) - return curses.wrapper(app.run) - - -def main(argv=None): - """ Main function / entry point """ - from discovery.shodan import Shodan - from discovery.shodan.cli.helpers import get_api_key - - api = Shodan(get_api_key()) - return launch_map(api) - -if __name__ == '__main__': - import sys - sys.exit(main()) diff --git a/discovery/shodan/client.py b/discovery/shodan/client.py deleted file mode 100644 index dfe293b7..00000000 --- a/discovery/shodan/client.py +++ /dev/null @@ -1,508 +0,0 @@ -# -*- coding: utf-8 -*- -""" -shodan.client -~~~~~~~~~~~~~ - -This module implements the Shodan API. - -:copyright: (c) 2014- by John Matherly -""" -import time - -import requests -import json - -from .exception import APIError -from .helpers import api_request, create_facet_string -from .stream import Stream - - -# Try to disable the SSL warnings in urllib3 since not everybody can install -# C extensions. If you're able to install C extensions you can try to run: -# -# pip install requests[security] -# -# Which will download libraries that offer more full-featured SSL classes -try: - requests.packages.urllib3.disable_warnings() -except: - pass - -# Define a basestring type if necessary for Python3 compatibility -try: - str -except NameError: - basestring = str - - -class Shodan: - """Wrapper around the Shodan REST and Streaming APIs - - :param key: The Shodan API key that can be obtained from your account page (https://account.shodan.io) - :type key: str - :ivar exploits: An instance of `shodan.Shodan.Exploits` that provides access to the Exploits REST API. - :ivar stream: An instance of `shodan.Shodan.Stream` that provides access to the Streaming API. - """ - - class Data: - - def __init__(self, parent): - self.parent = parent - - def list_datasets(self): - """Returns a list of datasets that the user has permission to download. - - :returns: A list of objects where every object describes a dataset - """ - return self.parent._request('/shodan/data', {}) - - def list_files(self, dataset): - """Returns a list of files that belong to the given dataset. - - :returns: A list of objects where each object contains a 'name', 'size', 'timestamp' and 'url' - """ - return self.parent._request('/shodan/data/{}'.format(dataset), {}) - - class Tools: - - def __init__(self, parent): - self.parent = parent - - def myip(self): - """Get your current IP address as seen from the Internet. - - :returns: str -- your IP address - """ - return self.parent._request('/tools/myip', {}) - - class Exploits: - - def __init__(self, parent): - self.parent = parent - - def search(self, query, page=1, facets=None): - """Search the entire Shodan Exploits archive using the same query syntax - as the website. - - :param query: The exploit search query; same syntax as website. - :type query: str - :param facets: A list of strings or tuples to get summary information on. - :type facets: str - :param page: The page number to access. - :type page: int - :returns: dict -- a dictionary containing the results of the search. - """ - query_args = { - 'query': query, - 'page': page, - } - if facets: - query_args['facets'] = create_facet_string(facets) - - return self.parent._request('/api/search', query_args, service='exploits') - - def count(self, query, facets=None): - """Search the entire Shodan Exploits archive but only return the total # of results, - not the actual exploits. - - :param query: The exploit search query; same syntax as website. - :type query: str - :param facets: A list of strings or tuples to get summary information on. - :type facets: str - :returns: dict -- a dictionary containing the results of the search. - - """ - query_args = { - 'query': query, - } - if facets: - query_args['facets'] = create_facet_string(facets) - - return self.parent._request('/api/count', query_args, service='exploits') - - class Labs: - - def __init__(self, parent): - self.parent = parent - - def honeyscore(self, ip): - """Calculate the probability of an IP being an ICS honeypot. - - :param ip: IP address of the device - :type ip: str - - :returns: int -- honeyscore ranging from 0.0 to 1.0 - """ - return self.parent._request('/labs/honeyscore/{}'.format(ip), {}) - - def __init__(self, key): - """Initializes the API object. - - :param key: The Shodan API key. - :type key: str - """ - self.api_key = key - self.base_url = 'https://api.shodan.io' - self.base_exploits_url = 'https://exploits.shodan.io' - self.data = self.Data(self) - self.exploits = self.Exploits(self) - self.labs = self.Labs(self) - self.tools = self.Tools(self) - self.stream = Stream(key) - self._session = requests.Session() - - def _request(self, function, params, service='shodan', method='get'): - """General-purpose function to create web requests to SHODAN. - - Arguments: - function -- name of the function you want to execute - params -- dictionary of parameters for the function - - Returns - A dictionary containing the function's results. - - """ - # Add the API key parameter automatically - params['key'] = self.api_key - - # Determine the base_url based on which service we're interacting with - base_url = { - 'shodan': self.base_url, - 'exploits': self.base_exploits_url, - }.get(service, 'shodan') - - # Send the request - try: - if method.lower() == 'post': - data = self._session.post(base_url + function, params) - else: - data = self._session.get(base_url + function, params=params) - except: - raise APIError('Unable to connect to Shodan') - - # Check that the API key wasn't rejected - if data.status_code == 401: - try: - # Return the actual error message if the API returned valid JSON - error = data.json()['error'] - except Exception as e: - error = 'Invalid API key' - - raise APIError(error) - - # Parse the text into JSON - try: - data = data.json() - except: - raise APIError('Unable to parse JSON response') - - # Raise an exception if an error occurred - if type(data) == dict and 'error' in data: - raise APIError(data['error']) - - # Return the data - return data - - def count(self, query, facets=None): - """Returns the total number of search results for the query. - - :param query: Search query; identical syntax to the website - :type query: str - :param facets: (optional) A list of properties to get summary information on - :type facets: str - - :returns: A dictionary with 1 main property: total. If facets have been provided then another property called "facets" will be available at the top-level of the dictionary. Visit the website for more detailed information. - """ - query_args = { - 'query': query, - } - if facets: - query_args['facets'] = create_facet_string(facets) - return self._request('/shodan/host/count', query_args) - - def host(self, ips, history=False, minify=False): - """Get all available information on an IP. - - :param ip: IP of the computer - :type ip: str - :param history: (optional) True if you want to grab the historical (non-current) banners for the host, False otherwise. - :type history: bool - :param minify: (optional) True to only return the list of ports and the general host information, no banners, False otherwise. - :type minify: bool - """ - if isinstance(ips, str): - ips = [ips] - - params = {} - if history: - params['history'] = history - if minify: - params['minify'] = minify - return self._request('/shodan/host/%s' % ','.join(ips), params) - - def info(self): - """Returns information about the current API key, such as a list of add-ons - and other features that are enabled for the current user's API plan. - """ - return self._request('/api-info', {}) - - def ports(self): - """Get a list of ports that Shodan crawls - - :returns: An array containing the ports that Shodan crawls for. - """ - return self._request('/shodan/ports', {}) - - def protocols(self): - """Get a list of protocols that the Shodan on-demand scanning API supports. - - :returns: A dictionary containing the protocol name and description. - """ - return self._request('/shodan/protocols', {}) - - def scan(self, ips, force=False): - """Scan a network using Shodan - - :param ips: A list of IPs or netblocks in CIDR notation or an object structured like: - { - "9.9.9.9": [ - (443, "https"), - (8080, "http") - ], - "1.1.1.0/24": [ - (503, "modbus") - ] - } - :type ips: str or dict - :param force: Whether or not to force Shodan to re-scan the provided IPs. Only available to enterprise users. - :type force: bool - - :returns: A dictionary with a unique ID to check on the scan progress, the number of IPs that will be crawled and how many scan credits are left. - """ - if isinstance(ips, basestring): - ips = [ips] - - if isinstance(ips, dict): - networks = json.dumps(ips) - else: - networks = ','.join(ips) - - params = { - 'ips': networks, - 'force': force, - } - - return self._request('/shodan/scan', params, method='post') - - def scan_internet(self, port, protocol): - """Scan a network using Shodan - - :param port: The port that should get scanned. - :type port: int - :param port: The name of the protocol as returned by the protocols() method. - :type port: str - - :returns: A dictionary with a unique ID to check on the scan progress. - """ - params = { - 'port': port, - 'protocol': protocol, - } - - return self._request('/shodan/scan/internet', params, method='post') - - def scan_status(self, scan_id): - """Get the status information about a previously submitted scan. - - :param id: The unique ID for the scan that was submitted - :type id: str - - :returns: A dictionary with general information about the scan, including its status in getting processed. - """ - return self._request('/shodan/scan/%s' % scan_id, {}) - - def search(self, query, page=1, limit=None, offset=None, facets=None, minify=True): - """Search the SHODAN database. - - :param query: Search query; identical syntax to the website - :type query: str - :param page: (optional) Page number of the search results - :type page: int - :param limit: (optional) Number of results to return - :type limit: int - :param offset: (optional) Search offset to begin getting results from - :type offset: int - :param facets: (optional) A list of properties to get summary information on - :type facets: str - :param minify: (optional) Whether to minify the banner and only return the important data - :type minify: bool - - :returns: A dictionary with 2 main items: matches and total. If facets have been provided then another property called "facets" will be available at the top-level of the dictionary. Visit the website for more detailed information. - """ - args = { - 'query': query, - 'minify': minify, - } - if limit: - args['limit'] = limit - if offset: - args['offset'] = offset - else: - args['page'] = page - - if facets: - args['facets'] = create_facet_string(facets) - - return self._request('/shodan/host/search', args) - - def search_cursor(self, query, minify=True, retries=5): - """Search the SHODAN database. - - This method returns an iterator that can directly be in a loop. Use it when you want to loop over - all of the results of a search query. But this method doesn't return a "matches" array or the "total" - information. And it also can't be used with facets, it's only use is to iterate over results more - easily. - - :param query: Search query; identical syntax to the website - :type query: str - :param minify: (optional) Whether to minify the banner and only return the important data - :type minify: bool - :param retries: (optional) How often to retry the search in case it times out - :type minify: int - - :returns: A search cursor that can be used as an iterator/ generator. - """ - args = { - 'query': query, - 'minify': minify, - } - - page = 1 - tries = 0 - while page == 1 or results['matches']: - try: - results = self.search(query, minify=minify, page=page) - for banner in results['matches']: - try: - yield banner - except GeneratorExit: - return # exit out of the function - page += 1 - tries = 0 - except: - # We've retried several times but it keeps failing, so lets error out - if tries >= retries: - break - - tries += 1 - time.sleep(1.0) # wait 1 second if the search errored out for some reason - - def search_tokens(self, query): - """Returns information about the search query itself (filters used etc.) - - :param query: Search query; identical syntax to the website - :type query: str - - :returns: A dictionary with 4 main properties: filters, errors, attributes and string. - """ - query_args = { - 'query': query, - } - return self._request('/shodan/host/search/tokens', query_args) - - def services(self): - """Get a list of services that Shodan crawls - - :returns: A dictionary containing the ports/ services that Shodan crawls for. The key is the port number and the value is the name of the service. - """ - return self._request('/shodan/services', {}) - - def queries(self, page=1, sort='timestamp', order='desc'): - """List the search queries that have been shared by other users. - - :param page: Page number to iterate over results; each page contains 10 items - :type page: int - :param sort: Sort the list based on a property. Possible values are: votes, timestamp - :type sort: str - :param order: Whether to sort the list in ascending or descending order. Possible values are: asc, desc - :type order: str - - :returns: A list of saved search queries (dictionaries). - """ - args = { - 'page': page, - 'sort': sort, - 'order': order, - } - return self._request('/shodan/query', args) - - def queries_search(self, query, page=1): - """Search the directory of saved search queries in Shodan. - - :param query: The search string to look for in the search query - :type query: str - :param page: Page number to iterate over results; each page contains 10 items - :type page: int - - :returns: A list of saved search queries (dictionaries). - """ - args = { - 'page': page, - 'query': query, - } - return self._request('/shodan/query/search', args) - - def queries_tags(self, size=10): - """Search the directory of saved search queries in Shodan. - - :param query: The number of tags to return - :type page: int - - :returns: A list of tags. - """ - args = { - 'size': size, - } - return self._request('/shodan/query/tags', args) - - def create_alert(self, name, ip, expires=0): - """Search the directory of saved search queries in Shodan. - - :param query: The number of tags to return - :type page: int - - :returns: A list of tags. - """ - data = { - 'name': name, - 'filters': { - 'ip': ip, - }, - 'expires': expires, - } - - response = api_request(self.api_key, '/shodan/alert', data=data, params={}, method='post') - - return response - - def alerts(self, aid=None, include_expired=True): - """List all of the active alerts that the user created.""" - if aid: - func = '/shodan/alert/%s/info' % aid - else: - func = '/shodan/alert/info' - - response = api_request(self.api_key, func, params={ - 'include_expired': include_expired, - }) - - return response - - def delete_alert(self, aid): - """Delete the alert with the given ID.""" - func = '/shodan/alert/%s' % aid - - response = api_request(self.api_key, func, params={}, method='delete') - - return response - diff --git a/discovery/shodan/exception.py b/discovery/shodan/exception.py deleted file mode 100644 index 84ad41e2..00000000 --- a/discovery/shodan/exception.py +++ /dev/null @@ -1,20 +0,0 @@ -class WebAPIError(Exception): - def __init__(self, value): - self.value = value - - def __str__(self): - return self.value - - -class APIError(Exception): - """This exception gets raised whenever a non-200 status code was returned by the Shodan API.""" - - def __init__(self, value): - self.value = value - - def __str__(self): - return self.value - - -class APITimeout(APIError): - pass diff --git a/discovery/shodan/helpers.py b/discovery/shodan/helpers.py deleted file mode 100644 index fa78159a..00000000 --- a/discovery/shodan/helpers.py +++ /dev/null @@ -1,167 +0,0 @@ -import gzip -import requests -import json - -from .exception import APIError - -try: - str -except NameError: - basestring = str - - -def create_facet_string(facets): - """Converts a Python list of facets into a comma-separated string that can be understood by - the Shodan API. - """ - facet_str = '' - for facet in facets: - if isinstance(facet, str): - facet_str += facet - else: - facet_str += '%s:%s' % (facet[0], facet[1]) - facet_str += ',' - return facet_str[:-1] - - -def api_request(key, function, params=None, data=None, base_url='https://api.shodan.io', method='get', retries=1): - """General-purpose function to create web requests to SHODAN. - - Arguments: - function -- name of the function you want to execute - params -- dictionary of parameters for the function - - Returns - A dictionary containing the function's results. - - """ - # Add the API key parameter automatically - params['key'] = key - - # Send the request - tries = 0 - error = False - while tries <= retries: - try: - if method.lower() == 'post': - data = requests.post(base_url + function, json.dumps(data), params=params, headers={'content-type': 'application/json'}) - elif method.lower() == 'delete': - data = requests.delete(base_url + function, params=params) - else: - data = requests.get(base_url + function, params=params) - - # Exit out of the loop - break - except: - error = True - tries += 1 - - if error and tries >= retries: - raise APIError('Unable to connect to Shodan') - - # Check that the API key wasn't rejected - if data.status_code == 401: - try: - raise APIError(data.json()['error']) - except: - pass - raise APIError('Invalid API key') - - # Parse the text into JSON - try: - data = data.json() - except: - raise APIError('Unable to parse JSON response') - - # Raise an exception if an error occurred - if type(data) == dict and data.get('error', None): - raise APIError(data['error']) - - # Return the data - return data - - -def iterate_files(files, fast=False): - """Loop over all the records of the provided Shodan output file(s).""" - from json import loads - if fast: - # Try to use ujson for parsing JSON if it's available and the user requested faster throughput - # It's significantly faster at encoding/ decoding JSON but it doesn't support as - # many options as the standard library. As such, we're mostly interested in using it for - # decoding since reading/ parsing files will use up the most time. - try: - from ujson import loads - except: - pass - - if isinstance(files, str): - files = [files] - - for filename in files: - # Create a file handle depending on the filetype - if filename.endswith('.gz'): - fin = gzip.open(filename, 'r') - else: - fin = open(filename, 'r') - - for line in fin: - # Ensure the line has been decoded into a string to prevent errors w/ Python3 - line = line.decode('utf-8') - - # Convert the JSON into a native Python object - banner = loads(line) - yield banner - -def get_screenshot(banner): - if 'opts' in banner and 'screenshot' in banner['opts']: - return banner['opts']['screenshot'] - return None - - -def get_ip(banner): - if 'ipv6' in banner: - return banner['ipv6'] - return banner['ip_str'] - - -def open_file(filename, mode='a', compresslevel=9): - return gzip.open(filename, mode, compresslevel) - - -def write_banner(fout, banner): - line = json.dumps(banner) + '\n' - fout.write(line.encode('utf-8')) - - -def humanize_bytes(bytes, precision=1): - """Return a humanized string representation of a number of bytes. - >>> humanize_bytes(1) - '1 byte' - >>> humanize_bytes(1024) - '1.0 kB' - >>> humanize_bytes(1024*123) - '123.0 kB' - >>> humanize_bytes(1024*12342) - '12.1 MB' - >>> humanize_bytes(1024*12342,2) - '12.05 MB' - >>> humanize_bytes(1024*1234,2) - '1.21 MB' - >>> humanize_bytes(1024*1234*1111,2) - '1.31 GB' - >>> humanize_bytes(1024*1234*1111,1) - '1.3 GB' - """ - - if bytes == 1: - return '1 byte' - if bytes < 1024: - return '%.*f %s' % (precision, bytes, "bytes") - - suffixes = ['KB', 'MB', 'GB', 'TB', 'PB'] - multiple = 1024.0 #.0 force float on python 2 - for suffix in suffixes: - bytes /= multiple - if bytes < multiple: - return '%.*f %s' % (precision, bytes, suffix) - return '%.*f %s' % (precision, bytes, suffix) diff --git a/discovery/shodan/stream.py b/discovery/shodan/stream.py deleted file mode 100644 index 20e1f335..00000000 --- a/discovery/shodan/stream.py +++ /dev/null @@ -1,123 +0,0 @@ -import requests -import json -import ssl - -from .exception import APIError - - -class Stream: - - base_url = 'https://stream.shodan.io' - - def __init__(self, api_key): - self.api_key = api_key - - def _create_stream(self, name, timeout=None): - # The user doesn't want to use a timeout - # If the timeout is specified as 0 then we also don't want to have a timeout - if ( timeout and timeout <= 0 ) or ( timeout == 0 ): - timeout = None - - try: - while True: - req = requests.get(self.base_url + name, params={'key': self.api_key}, stream=True, timeout=timeout) - - # Status code 524 is special to Cloudflare - # It means that no data was sent from the streaming servers which caused Cloudflare - # to terminate the connection. - # - # We only want to exit if there was a timeout specified or the HTTP status code is - # not specific to Cloudflare. - if req.status_code != 524 or timeout >= 0: - break - except Exception as e: - raise APIError('Unable to contact the Shodan Streaming API') - - if req.status_code != 200: - try: - data = json.loads(req.text) - raise APIError(data['error']) - except APIError as e: - raise - except Exception as e: - pass - raise APIError('Invalid API key or you do not have access to the Streaming API') - if req.encoding is None: - req.encoding = 'utf-8' - return req - - def _iter_stream(self, stream, raw, timeout=None): - for line in stream.iter_lines(decode_unicode=True): - # The Streaming API sends out heartbeat messages that are newlines - # We want to ignore those messages since they don't contain any data - if line: - if raw: - yield line - else: - yield json.loads(line) - else: - # If the user specified a timeout then we want to keep track of how long we've - # been getting heartbeat messages and exit the loop if it's been too long since - # we've seen any activity. - if timeout: - # TODO: This is a placeholder for now but since the Streaming API added heartbeats it broke - # the ability to use inactivity timeouts (the connection timeout still works). The timeout is - # mostly needed when doing on-demand scans and wanting to temporarily consume data from a - # network alert. - pass - - def alert(self, aid=None, timeout=None, raw=False): - if aid: - stream = self._create_stream('/shodan/alert/%s' % aid, timeout=timeout) - else: - stream = self._create_stream('/shodan/alert', timeout=timeout) - - try: - for line in self._iter_stream(stream, raw): - yield line - except requests.exceptions.ConnectionError as e: - raise APIError('Stream timed out') - except ssl.SSLError as e: - raise APIError('Stream timed out') - - def asn(self, asn, raw=False, timeout=None): - """ - A filtered version of the "banners" stream to only return banners that match the ASNs of interest. - - :param asn: A list of ASN to return banner data on. - :type asn: string[] - """ - stream = self._create_stream('/shodan/asn/%s' % ','.join(asn), timeout=timeout) - for line in self._iter_stream(stream, raw): - yield line - - def banners(self, raw=False, timeout=None): - """A real-time feed of the data that Shodan is currently collecting. Note that this is only available to - API subscription plans and for those it only returns a fraction of the data. - """ - stream = self._create_stream('/shodan/banners', timeout=timeout) - for line in self._iter_stream(stream, raw): - yield line - - def countries(self, countries, raw=False, timeout=None): - """ - A filtered version of the "banners" stream to only return banners that match the countries of interest. - - :param countries: A list of countries to return banner data on. - :type countries: string[] - """ - stream = self._create_stream('/shodan/countries/%s' % ','.join(countries), timeout=timeout) - for line in self._iter_stream(stream, raw): - yield line - - def ports(self, ports, raw=False, timeout=None): - """ - A filtered version of the "banners" stream to only return banners that match the ports of interest. - - :param ports: A list of ports to return banner data on. - :type ports: int[] - """ - stream = self._create_stream('/shodan/ports/%s' % ','.join([str(port) for port in ports]), timeout=timeout) - for line in self._iter_stream(stream, raw): - yield line - diff --git a/discovery/shodan/threatnet.py b/discovery/shodan/threatnet.py deleted file mode 100644 index b458dead..00000000 --- a/discovery/shodan/threatnet.py +++ /dev/null @@ -1,66 +0,0 @@ -import requests -import json - -from .exception import APIError - - -class Threatnet: - """Wrapper around the Threatnet REST and Streaming APIs - - :param key: The Shodan API key that can be obtained from your account page (https://account.shodan.io) - :type key: str - :ivar stream: An instance of `shodan.Threatnet.Stream` that provides access to the Streaming API. - """ - - class Stream: - - base_url = 'https://stream.shodan.io' - - def __init__(self, parent): - self.parent = parent - - def _create_stream(self, name): - try: - req = requests.get(self.base_url + name, params={'key': self.parent.api_key}, stream=True) - except: - raise APIError('Unable to contact the Shodan Streaming API') - - if req.status_code != 200: - try: - raise APIError(data.json()['error']) - except: - pass - raise APIError('Invalid API key or you do not have access to the Streaming API') - return req - - def events(self): - stream = self._create_stream('/threatnet/events') - for line in stream.iter_lines(): - if line: - banner = json.loads(line) - yield banner - - def backscatter(self): - stream = self._create_stream('/threatnet/backscatter') - for line in stream.iter_lines(): - if line: - banner = json.loads(line) - yield banner - - def activity(self): - stream = self._create_stream('/threatnet/ssh') - for line in stream.iter_lines(): - if line: - banner = json.loads(line) - yield banner - - def __init__(self, key): - """Initializes the API object. - - :param key: The Shodan API key. - :type key: str - """ - self.api_key = key - self.base_url = 'https://api.shodan.io' - self.stream = self.Stream(self) - diff --git a/discovery/shodansearch.py b/discovery/shodansearch.py index 760fbb25..7523c672 100644 --- a/discovery/shodansearch.py +++ b/discovery/shodansearch.py @@ -1,24 +1,43 @@ -from discovery.shodan import Shodan +from shodan import Shodan +from shodan import exception +#from discovery.shodan import Shodan from discovery.constants import * -class search_shodan(): +class search_shodan: - def __init__(self, host): - self.host = host + def __init__(self): self.key = shodanAPI_key if self.key == "": raise MissingKey(True) self.api = Shodan(self.key) + self.hostdatarow = [] - def run(self): + def search_ip(self, ip): try: - result = self.api.host(self.host) - # for service in result['data']: - # print ("%s:%s" % (service['ip_str'], service['port'])) - # print ("%s" % (service['product'])) - # print ("%s" % (service['hostnames'])) - return result + ipaddress = ip + results = self.api.host(ipaddress) + technologies = [] + servicesports = [] + for result in results['data']: + try: + for key in result['http']['components'].keys(): + technologies.append(key) + except KeyError as e: + pass + port = str(result.get('port')) + product = str(result.get('product')) + servicesports.append(str(product)+':'+str(port)) + technologies = list(set(technologies)) + self.hostdatarow = [ + str(results.get('ip_str')), str(results.get('hostnames')).strip('[]\''), + str(results.get('org')), str(servicesports).strip('[]\''), + str(technologies).strip('[]\'')] + except exception.APIError: + print(ipaddress+": Not in Shodan") + self.hostdatarow = [ipaddress, "Not in Shodan", "Not in Shodan", "Not in Shodan", "Not in Shodan"] + except Exception as e: - print("SHODAN empty reply or error in the call") - return "error" \ No newline at end of file + print("Error occurred in the Shodan IP search module: " + str(e)) + finally: + return self.hostdatarow \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 4ce044bc..344ed4fc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,5 @@ beautifulsoup4>=4.7.0 plotly>=3.4.2 -requests>=2.21.0 \ No newline at end of file +requests>=2.21.0 +texttable>=1.4.0 +shodan>=1.10.0 \ No newline at end of file diff --git a/theHarvester.py b/theHarvester.py index 5340e78e..65d649c6 100755 --- a/theHarvester.py +++ b/theHarvester.py @@ -4,6 +4,7 @@ import getopt import re import stash import datetime +import time from discovery import * from discovery.constants import * from lib.core import * @@ -139,7 +140,7 @@ def start(argv): print("[-] Searching in Censys.") from discovery import censys # Import locally or won't work - search = censys.search_censys(word,limit) + search = censys.search_censys(word, limit) search.process() all_ip = search.get_ipaddresses() hosts = filter(search.get_hostnames()) @@ -194,7 +195,8 @@ def start(argv): elif engineitem == "googleCSE": print("[-] Searching in Google Custom Search.") try: - search = googleCSE.search_googleCSE(word, limit, start) + search = googleCSE.search_googleCSE( + word, limit, start) search.process() search.store_results() all_emails = filter(search.get_emails()) @@ -211,13 +213,16 @@ def start(argv): pass elif engineitem == "google-certificates": - print("[-] Searching in Google Certificate transparency report.") - search = googlecertificates.search_googlecertificates(word, limit, start) + print( + "[-] Searching in Google Certificate transparency report.") + search = googlecertificates.search_googlecertificates( + word, limit, start) search.process() hosts = filter(search.get_domains()) all_hosts.extend(hosts) db = stash.stash_manager() - db.store_all(word, all_hosts, 'host', 'google-certificates') + db.store_all(word, all_hosts, 'host', + 'google-certificates') elif engineitem == "google-profiles": print("[-] Searching in Google profiles.") @@ -237,7 +242,8 @@ def start(argv): from discovery import huntersearch # Import locally or won't work. try: - search = huntersearch.search_hunter(word, limit, start) + search = huntersearch.search_hunter( + word, limit, start) search.process() emails = filter(search.get_emails()) all_emails.extend(emails) @@ -292,7 +298,8 @@ def start(argv): print("[-] Searching in SecurityTrails.") from discovery import securitytrailssearch try: - search = securitytrailssearch.search_securitytrail(word) + search = securitytrailssearch.search_securitytrail( + word) search.process() hosts = filter(search.get_hostnames()) all_hosts.extend(hosts) @@ -316,7 +323,8 @@ def start(argv): hosts = filter(search.get_hostnames()) all_hosts.extend(hosts) db = stash.stash_manager() - db.store_all(word, all_hosts, 'host', 'threatcrowd') + db.store_all(word, all_hosts, + 'host', 'threatcrowd') except Exception: pass @@ -394,7 +402,7 @@ def start(argv): print("[-] Searching in Censys.") from discovery import censys - search = censys.search_censys(word,limit) + search = censys.search_censys(word, limit) search.process() ips = search.get_ipaddresses() setips = set(ips) @@ -432,13 +440,16 @@ def start(argv): db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'google') - print("[-] Searching in Google Certificate transparency report.") - search = googlecertificates.search_googlecertificates(word, limit, start) + print( + "[-] Searching in Google Certificate transparency report.") + search = googlecertificates.search_googlecertificates( + word, limit, start) search.process() domains = filter(search.get_domains()) all_hosts.extend(domains) db = stash.stash_manager() - db.store_all(word, all_hosts, 'host', 'google-certificates') + db.store_all(word, all_hosts, 'host', + 'google-certificates') # googleplus @@ -450,7 +461,8 @@ def start(argv): from discovery import huntersearch # Import locally. try: - search = huntersearch.search_hunter(word, limit, start) + search = huntersearch.search_hunter( + word, limit, start) search.process() emails = filter(search.get_emails()) hosts = filter(search.get_hostnames()) @@ -500,7 +512,8 @@ def start(argv): hosts = filter(search.get_hostnames()) all_hosts.extend(hosts) db = stash.stash_manager() - db.store_all(word, all_hosts, 'host', 'threatcrowd') + db.store_all(word, all_hosts, + 'host', 'threatcrowd') except Exception: pass @@ -657,7 +670,8 @@ def start(argv): s = '.' range = s.join(range) if not analyzed_ranges.count(range): - print(("\033[94m[-] Performing reverse lookup in " + range + "\033[1;33;40m")) + print( + ("\033[94m[-] Performing reverse lookup in " + range + "\033[1;33;40m")) a = dnssearch.dns_reverse(range, True) a.list() res = a.process() @@ -709,29 +723,31 @@ def start(argv): # Shodan search shodanres = [] - shodanvisited = [] + import texttable + tab = texttable.Texttable() + header = ["IP address", "Hostname", "Org", + "Services:Ports", "Technologies"] + tab.header(header) + tab.set_cols_align(["c", "c", "c", "c", "c"]) + tab.set_cols_valign(["m", "m", "m", "m", "m"]) + tab.set_chars(['-', '|', '+', '#']) + tab.set_cols_width([15, 20, 15, 15, 18]) + host_ip = list(set(host_ip)) if shodan is True: print("\n\n\033[1;32;40m[-] Shodan DB search (passive):\n") - if full == []: - print('No host to search, exiting.') - sys.exit(1) - for x in full: - try: - ip = x.split(":")[1] - if not shodanvisited.count(ip): - print(("\tSearching for: " + ip)) - a = shodansearch.search_shodan(ip) - shodanvisited.append(ip) - results = a.run() - for res in results['data']: - shodanres.append( - str("%s:%s - %s - %s - %s," % (res['ip_str'], res['port'], res['os'], res['isp']))) - except Exception as e: - pass - print("\n [+] Shodan results:") - print("-------------------") - for x in shodanres: - print(x) + try: + for ip in host_ip: + print(("\tSearching for: " + ip)) + shodan = shodansearch.search_shodan() + rowdata = shodan.search_ip(ip) + time.sleep(2) + tab.add_row(rowdata) + printedtable = tab.draw() + print("\n [+] Shodan results:") + print("-------------------") + print(printedtable) + except Exception as e: + print("Error occurred in theHarvester - Shodan search module: " + str(e)) else: pass @@ -757,18 +773,22 @@ def start(argv): db = stash.stash_manager() scanboarddata = db.getscanboarddata() latestscanresults = db.getlatestscanresults(word) - previousscanresults = db.getlatestscanresults(word, previousday=True) + previousscanresults = db.getlatestscanresults( + word, previousday=True) latestscanchartdata = db.latestscanchartdata(word) scanhistorydomain = db.getscanhistorydomain(word) pluginscanstatistics = db.getpluginscanstatistics() generator = statichtmlgenerator.htmlgenerator(word) HTMLcode = generator.beginhtml() HTMLcode += generator.generatelatestscanresults(latestscanresults) - HTMLcode += generator.generatepreviousscanresults(previousscanresults) + HTMLcode += generator.generatepreviousscanresults( + previousscanresults) graph = reportgraph.graphgenerator(word) HTMLcode += graph.drawlatestscangraph(word, latestscanchartdata) - HTMLcode += graph.drawscattergraphscanhistory(word, scanhistorydomain) - HTMLcode += generator.generatepluginscanstatistics(pluginscanstatistics) + HTMLcode += graph.drawscattergraphscanhistory( + word, scanhistorydomain) + HTMLcode += generator.generatepluginscanstatistics( + pluginscanstatistics) HTMLcode += generator.generatedashboardcode(scanboarddata) HTMLcode += '

    Report generated on ' + str( datetime.datetime.now()) + '

    ' @@ -804,13 +824,15 @@ def start(argv): for x in full: x = x.split(":") if len(x) == 2: - file.write('' + '' + x[1] + '' + x[0] + '' + '') + file.write( + '' + '' + x[1] + '' + x[0] + '' + '') else: file.write('' + x + '') for x in vhost: x = x.split(":") if len(x) == 2: - file.write('' + '' + x[1] + '' + x[0] + '' + '') + file.write( + '' + '' + x[1] + '' + x[0] + '' + '') else: file.write('' + x + '') if shodanres != []: From 4e270874f013e5b6d93a7a87e7f8e4ee53dbfb97 Mon Sep 17 00:00:00 2001 From: Janos Zold Date: Fri, 4 Jan 2019 16:34:54 +0000 Subject: [PATCH 2/3] update docker file with dependencies --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e483b7e9..07a9b6c4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM python:2-alpine RUN mkdir /app -RUN pip install requests beautifulsoup4 texttable +RUN pip install requests beautifulsoup4 texttable plotly shodan WORKDIR /app COPY . /app RUN chmod +x *.py From 58625987a0ad29d136aabbf4ef3a6594c2c2c8fa Mon Sep 17 00:00:00 2001 From: Janos Zold Date: Fri, 4 Jan 2019 17:26:00 +0000 Subject: [PATCH 3/3] remove quotes from Shodan table --- discovery/shodansearch.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/discovery/shodansearch.py b/discovery/shodansearch.py index 7523c672..9750e7db 100644 --- a/discovery/shodansearch.py +++ b/discovery/shodansearch.py @@ -31,8 +31,8 @@ class search_shodan: technologies = list(set(technologies)) self.hostdatarow = [ str(results.get('ip_str')), str(results.get('hostnames')).strip('[]\''), - str(results.get('org')), str(servicesports).strip('[]\''), - str(technologies).strip('[]\'')] + str(results.get('org')), str(servicesports).replace('\'', '').strip('[]'), + str(technologies).replace('\'', '').strip('[]')] except exception.APIError: print(ipaddress+": Not in Shodan") self.hostdatarow = [ipaddress, "Not in Shodan", "Not in Shodan", "Not in Shodan", "Not in Shodan"]