diff --git a/.github/workflows/theHarvester.yml b/.github/workflows/theHarvester.yml
index c006e4e9..eb29e14f 100644
--- a/.github/workflows/theHarvester.yml
+++ b/.github/workflows/theHarvester.yml
@@ -26,9 +26,57 @@ jobs:
- name: Install dependencies
run: |
pip install -r requirements.txt
- - name: Run theHarvester
+ - name: Run theHarvester module baidu
run: |
- python theHarvester.py -d metasploit.com -b baidu,bing,censys,crtsh,dnsdumpster,dogpile,duckduckgo,exalead,linkedin,netcraft,threatcrowd,trello,twitter,virustotal,yahoo
+ python theHarvester.py -d metasploit.com -b baidu
+ - name: Run theHarvester module bing
+ run: |
+ python theHarvester.py -d metasploit.com -b bing
+ - name: Run theHarvester module censys
+ run: |
+ python theHarvester.py -d metasploit.com -b censys
+ - name: Run theHarvester module crtsh
+ run: |
+ python theHarvester.py -d metasploit.com -b crtsh
+ - name: Run theHarvester module dnsdumpster
+ run: |
+ python theHarvester.py -d metasploit.com -b dnsdumpster
+ - name: Run theHarvester module dogplie
+ run: |
+ python theHarvester.py -d metasploit.com -b dogpile
+ - name: Run theHarvester module duckduckgo
+ run: |
+ python theHarvester.py -d metasploit.com -b duckduckgo
+ - name: Run theHarvester module exalead
+ run: |
+ python theHarvester.py -d metasploit.com -b exalead
+ - name: Run theHarvester module google
+ run: |
+ python theHarvester.py -d metasploit.com -b google
+ - name: Run theHarvester module linkedin
+ run: |
+ python theHarvester.py -d metasploit.com -b linkedin
+ - name: Run theHarvester module linkedin_links
+ run: |
+ python theHarvester.py -d metasploit.com -b linkedin_links
+ - name: Run theHarvester module netcraft
+ run: |
+ python theHarvester.py -d metasploit.com -b netcraft
+ - name: Run theHarvester module threatcrowd
+ run: |
+ python theHarvester.py -d metasploit.com -b threatcrowd
+ - name: Run theHarvester module trello
+ run: |
+ python theHarvester.py -d metasploit.com -b trello
+ - name: Run theHarvester module twitter
+ run: |
+ python theHarvester.py -d metasploit.com -b twitter
+ - name: Run theHarvester module virustotal
+ run: |
+ python theHarvester.py -d metasploit.com -b virustotal
+ - name: Run theHarvester module yahoo
+ run: |
+ python theHarvester.py -d metasploit.com -b yahoo
- name: Lint with flake8
run: |
# stop the build if there are Python syntax errors or undefined names
diff --git a/.gitignore b/.gitignore
index 83c17fa5..30769947 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,3 +8,5 @@ api-keys.yaml
debug_results.txt
tests/myparser.py
venv
+.mypy_cache
+.pytest_cache
diff --git a/.github/contributing.md b/CONTRIBUTING.md
similarity index 67%
rename from .github/contributing.md
rename to CONTRIBUTING.md
index f6f816d5..ef5fd561 100644
--- a/.github/contributing.md
+++ b/CONTRIBUTING.md
@@ -1,6 +1,5 @@
# Contributing to theHarvester Project
Welcome to theHarvester project, so you would like to contribute.
-
The following below must be met to get accepted.
# CI
@@ -12,7 +11,9 @@ For new modules a unit test for that module is required and we use pytest.
# Coding Standards
* No single letter variables and variable names must represent the action that it is performing
* Have static typing on functions etc
+* Make sure no errors are reported from mypy
+* No issues reported with flake8
# Submitting Bugs
-If you have a bug in a module that you want to submit an issue for and know how to write python code.
-Please create a unit test for that bug and submit a fix for it
\ No newline at end of file
+If you find a bug in a module that you want to submit an issue for and know how to write python code.
+Please create a unit test for that bug(If possible) and submit a fix for it as it would be a big help to the project.
\ No newline at end of file
diff --git a/README.md b/README.md
index c1631fee..91941510 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,5 @@
-
[](https://travis-ci.com/laramies/theHarvester) [](https://lgtm.com/projects/g/laramies/theHarvester/context:python)
[](https://inventory.rawsec.ml/)
@@ -67,7 +66,7 @@ Active:
-------
* DNS brute force: dictionary brute force enumeration
* DNS reverse lookup: reverse lookup of IP´s discovered in order to find hostnames
-* DNS TDL expansion: TLD dictionary brute force enumeration
+* DNS TLD expansion: TLD dictionary brute force enumeration
Modules that require an API key:
--------------------------------
diff --git a/tests/discovery/test_linkedin_links.py b/tests/discovery/test_linkedin_links.py
index 2dc6f3dc..b0c710c0 100644
--- a/tests/discovery/test_linkedin_links.py
+++ b/tests/discovery/test_linkedin_links.py
@@ -1,17 +1,43 @@
#!/usr/bin/env python3
# coding=utf-8
from theHarvester.discovery import linkedinsearch
+from theHarvester.discovery.constants import splitter
import pytest
+import os
+import re
class TestGetLinks(object):
+ def test_splitter(self):
+ results = [
+ 'https://www.linkedin.com/in/don-draper-b1045618',
+ 'https://www.linkedin.com/in/don-draper-b59210a',
+ 'https://www.linkedin.com/in/don-draper-b5bb50b3',
+ 'https://www.linkedin.com/in/don-draper-b83ba26',
+ 'https://www.linkedin.com/in/don-draper-b854a51'
+ ]
+ filtered_results = splitter(results)
+ assert len(filtered_results) == 1
+
def test_get_links(self):
search = linkedinsearch.SearchLinkedin("facebook.com", '100')
search.process()
links = search.get_links()
- for link in links:
- print(link)
+ assert type(links) == list
+
+ def test_links_linkedin(self):
+ dir_path = os.path.dirname(os.path.realpath(__file__))
+ mock_response = open(dir_path + "/test_linkedin_links.txt")
+ mock_response_content = mock_response.read()
+ mock_response.close()
+ reg_links = re.compile(r"url=https:\/\/www\.linkedin.com(.*?)&")
+ temp = reg_links.findall(mock_response_content)
+ resul = []
+ for regex_item in temp:
+ stripped_url = regex_item.replace("url=", "")
+ resul.append("https://www.linkedin.com" + stripped_url)
+ assert set(resul)
if __name__ == '__main__':
diff --git a/tests/discovery/test_linkedin_links.txt b/tests/discovery/test_linkedin_links.txt
new file mode 100644
index 00000000..b8804830
--- /dev/null
+++ b/tests/discovery/test_linkedin_links.txt
@@ -0,0 +1,5 @@
+LinkedIn
View NIKUL www.facebook.com/nikulact's profile on LinkedIn, the world's largest
+professional community. NIKUL has 1 job listed on their profile. See the ...
United States - http://www.facebook.com/pages/elkhorbat/302997479939
View elkhorbat lkhorbat's profile on LinkedIn, the world's largest professional
+community. elkhorbat has 1 job listed on their profile. See the complete profile on
diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py
index 533d9da6..9cf0b2fc 100644
--- a/theHarvester/__main__.py
+++ b/theHarvester/__main__.py
@@ -38,7 +38,6 @@ def start():
hunter, intelx,
linkedin, linkedin_links, netcraft, securityTrails, threatcrowd,
trello, twitter, vhost, virustotal, yahoo''')
- parser.add_argument('-x', '--exclude', help='exclude options when using all sources', type=str)
args = parser.parse_args()
try:
diff --git a/theHarvester/discovery/constants.py b/theHarvester/discovery/constants.py
index 7549c4c6..0fbe8c91 100644
--- a/theHarvester/discovery/constants.py
+++ b/theHarvester/discovery/constants.py
@@ -4,6 +4,31 @@ import random
googleUA = 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1464.0 Safari/537.36'
+def splitter(links):
+ """
+ Method that tries to remove duplicates
+ LinkedinLists pulls a lot of profiles with the same name.
+ This method triest to remove duplicates from the list.
+ :param links: list of links to remove duplicates from
+ :return: unique-ish list
+ """
+ unique_list = []
+ name_check = []
+ for url in links:
+ tail = url.split("/")[-1]
+ if len(tail) == 2 or tail == "zh-cn":
+ tail = url.split("/")[-2]
+ name = tail.split("-")
+ if len(name) > 1:
+ joined_name = name[0] + name[1]
+ else:
+ joined_name = name[0]
+ if joined_name not in name_check:
+ unique_list.append(url)
+ name_check.append(joined_name)
+ return unique_list
+
+
def filter(lst):
"""
Method that filters list
diff --git a/theHarvester/discovery/linkedinsearch.py b/theHarvester/discovery/linkedinsearch.py
index 22a58699..ef774df3 100644
--- a/theHarvester/discovery/linkedinsearch.py
+++ b/theHarvester/discovery/linkedinsearch.py
@@ -36,7 +36,7 @@ class SearchLinkedin:
def get_links(self):
links = myparser.Parser(self.totalresults, self.word)
- return links.links_linkedin()
+ return splitter(links.links_linkedin())
def process(self):
while self.counter < self.limit:
diff --git a/theHarvester/lib/markup.py b/theHarvester/lib/markup.py
index c749b75a..7466ab99 100644
--- a/theHarvester/lib/markup.py
+++ b/theHarvester/lib/markup.py
@@ -1,16 +1,19 @@
-# This code is in the public domain, it comes with absolutely no
-# warranty and you can do absolutely whatever you want with it.
+# This code is in the public domain, it comes
+# with absolutely no warranty and you can do
+# absolutely whatever you want with it.
+# type: ignore
-__date__ = '17 May 2007'
-__version__ = '1.7'
+__date__ = '16 March 2015'
+__version__ = '1.10'
__doc__ = """
This is markup.py - a Python module that attempts to
make it easier to generate HTML/XML from a Python program
in an intuitive, lightweight, customizable and pythonic way.
+It works with both python 2 and 3.
The code is in the public domain.
-Version: {0} as of {1}.
+Version: %s as of %s.
Documentation and further info is at http://markup.sourceforge.net/
@@ -18,37 +21,47 @@ Please send bug reports, feature requests, enhancement
ideas or questions to nogradi at gmail dot com.
Installation: drop markup.py somewhere into your Python path.
-""".format(__version__, __date__)
+""" % (__version__, __date__)
+
+
+basestring = str
+String = str
+long = int
+
+# tags which are reserved python keywords will be referred
+# to by a leading underscore otherwise we end up with a syntax error
+import keyword
class Element:
-
"""This class handles the addition of a new element."""
def __init__(self, tag, case='lower', parent=None):
self.parent = parent
- if case == 'lower':
- self.tag = tag.lower()
- else:
+ if case == 'upper':
self.tag = tag.upper()
+ elif case == 'lower':
+ self.tag = tag.lower()
+ elif case == 'given':
+ self.tag = tag
+ else:
+ self.tag = tag
def __call__(self, *args, **kwargs):
if len(args) > 1:
raise ArgumentError(self.tag)
- # If class_ was defined in parent, it should be added to every element.
+ # if class_ was defined in parent it should be added to every element
if self.parent is not None and self.parent.class_ is not None:
if 'class_' not in kwargs:
kwargs['class_'] = self.parent.class_
if self.parent is None and len(args) == 1:
- x = [self.render(self.tag, False, myarg, mydict)
- for myarg, mydict in _argsdicts(args, kwargs)]
+ x = [self.render(self.tag, False, myarg, mydict) for myarg, mydict in _argsdicts(args, kwargs)]
return '\n'.join(x)
elif self.parent is None and len(args) == 0:
- x = [self.render(self.tag, True, myarg, mydict)
- for myarg, mydict in _argsdicts(args, kwargs)]
+ x = [self.render(self.tag, True, myarg, mydict) for myarg, mydict in _argsdicts(args, kwargs)]
return '\n'.join(x)
if self.tag in self.parent.twotags:
@@ -57,8 +70,7 @@ class Element:
elif self.tag in self.parent.onetags:
if len(args) == 0:
for myarg, mydict in _argsdicts(args, kwargs):
- # Here myarg is always None, because len( args ) = 0.
- self.render(self.tag, True, myarg, mydict)
+ self.render(self.tag, True, myarg, mydict) # here myarg is always None, because len( args ) = 0
else:
raise ClosingError(self.tag)
elif self.parent.mode == 'strict_html' and self.tag in self.parent.deptags:
@@ -70,13 +82,10 @@ class Element:
"""Append the actual tags to content."""
out = "<%s" % tag
- for key, value in kwargs.items():
- # When value is None, that means stuff like <... checked>.
- if value is not None:
- # Strip this so class_ will mean class, etc.
- key = key.strip('_')
- # Special cases, maybe change _ to - overall?
- if key == 'http_equiv':
+ for key, value in list(kwargs.items()):
+ if value is not None: # when value is None that means stuff like <... checked>
+ key = key.strip('_') # strip this so class_ will mean class, etc.
+ if key == 'http_equiv': # special cases, maybe change _ to - overall?
key = 'http-equiv'
elif key == 'accept_charset':
key = 'accept-charset'
@@ -116,11 +125,7 @@ class Element:
class Page:
- """This is our main class representing a document. Elements are added as
- attributes of an instance of this class."""
-
- def __init__(self, mode='strict_html', case='lower',
- onetags=None, twotags=None, separator='\n', class_=None):
+ def __init__(self, mode='strict_html', case='lower', onetags=None, twotags=None, separator='\n', class_=None):
"""Stuff that effects the whole document.
mode -- 'strict_html' for HTML 4.01 (default)
@@ -130,6 +135,7 @@ class Page:
case -- 'lower' element names will be printed in lower case (default)
'upper' they will be printed in upper case
+ 'given' element names will be printed as they are given
onetags -- list or tuple of valid elements with opening tags only
twotags -- list or tuple of valid elements with both opening and closing tags
@@ -141,36 +147,16 @@ class Page:
class_ -- a class that will be added to every element if defined"""
- valid_onetags = [
- "AREA",
- "BASE",
- "BR",
- "COL",
- "FRAME",
- "HR",
- "IMG",
- "INPUT",
- "LINK",
- "META",
- "PARAM"]
- valid_twotags = [
- "A", "ABBR", "ACRONYM", "ADDRESS", "B", "BDO", "BIG", "BLOCKQUOTE", "BODY", "BUTTON",
- "CAPTION", "CITE", "CODE", "COLGROUP", "DD", "DEL", "DFN", "DIV", "DL", "DT", "EM", "FIELDSET",
- "FORM", "FRAMESET", "H1", "H2", "H3", "H4", "H5", "H6", "HEAD", "HTML", "I", "IFRAME", "INS",
- "KBD", "LABEL", "LEGEND", "LI", "MAP", "NOFRAMES", "NOSCRIPT", "OBJECT", "OL", "OPTGROUP",
- "OPTION", "P", "PRE", "Q", "SAMP", "SCRIPT", "SELECT", "SMALL", "SPAN", "STRONG", "STYLE",
- "SUB", "SUP", "TABLE", "TBODY", "TD", "TEXTAREA", "TFOOT", "TH", "THEAD", "TITLE", "TR",
- "TT", "UL", "VAR"]
+ valid_onetags = ["AREA", "BASE", "BR", "COL", "FRAME", "HR", "IMG", "INPUT", "LINK", "META", "PARAM"]
+ valid_twotags = ["A", "ABBR", "ACRONYM", "ADDRESS", "B", "BDO", "BIG", "BLOCKQUOTE", "BODY", "BUTTON",
+ "CAPTION", "CITE", "CODE", "COLGROUP", "DD", "DEL", "DFN", "DIV", "DL", "DT", "EM", "FIELDSET",
+ "FORM", "FRAMESET", "H1", "H2", "H3", "H4", "H5", "H6", "HEAD", "HTML", "I", "IFRAME", "INS",
+ "KBD", "LABEL", "LEGEND", "LI", "MAP", "NOFRAMES", "NOSCRIPT", "OBJECT", "OL", "OPTGROUP",
+ "OPTION", "P", "PRE", "Q", "SAMP", "SCRIPT", "SELECT", "SMALL", "SPAN", "STRONG", "STYLE",
+ "SUB", "SUP", "TABLE", "TBODY", "TD", "TEXTAREA", "TFOOT", "TH", "THEAD", "TITLE", "TR",
+ "TT", "UL", "VAR"]
deprecated_onetags = ["BASEFONT", "ISINDEX"]
- deprecated_twotags = [
- "APPLET",
- "CENTER",
- "DIR",
- "FONT",
- "MENU",
- "S",
- "STRIKE",
- "U"]
+ deprecated_twotags = ["APPLET", "CENTER", "DIR", "FONT", "MENU", "S", "STRIKE", "U"]
self.header = []
self.content = []
@@ -178,23 +164,23 @@ class Page:
self.case = case
self.separator = separator
- # init( ) sets it to True so we know that