diff --git a/README.md b/README.md index f358d35..a6276f8 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ # [![Generic badge](https://img.shields.io/badge/dynamic/json.svg?url=https://raw.githubusercontent.com/qeeqbox/social-analyzer/main/info&label=version&query=$.version&colorB=blue&style=flat-square)](https://github.com/qeeqbox/social-analyzer/blob/main/info) [![Generic badge](https://img.shields.io/badge/dynamic/json.svg?url=https://raw.githubusercontent.com/qeeqbox/social-analyzer/main/info&label=verified%20sites&query=$.websites&colorB=blue&style=flat-square)](https://github.com/qeeqbox/social-analyzer/blob/main/info) [![Generic badge](https://img.shields.io/badge/dynamic/json.svg?url=https://raw.githubusercontent.com/qeeqbox/social-analyzer/main/info&label=verified%20detections&query=$.detections&colorB=blue&style=flat-square)](https://github.com/qeeqbox/social-analyzer/blob/main/info) [![Generic badge](https://img.shields.io/badge/dynamic/json.svg?url=https://raw.githubusercontent.com/qeeqbox/social-analyzer/main/info&label=build&query=$.build&colorB=green&style=flat-square)](https://github.com/qeeqbox/social-analyzer/blob/main/info) [![Generic badge](https://img.shields.io/badge/dynamic/json.svg?url=https://raw.githubusercontent.com/qeeqbox/social-analyzer/main/info&label=test&query=$.test&colorB=green&style=flat-square)](https://github.com/qeeqbox/social-analyzer/blob/main/info) [![Generic badge](https://img.shields.io/badge/dynamic/json.svg?url=https://raw.githubusercontent.com/qeeqbox/social-analyzer/main/info&label=docker&query=$.docker&colorB=green&style=flat-square)](https://github.com/qeeqbox/social-analyzer/blob/main/info) [![Generic badge](https://img.shields.io/static/v1?label=%F0%9F%91%8D&message=!&color=yellow&style=flat-square)](https://github.com/qeeqbox/social-analyzer/stargazers) -Social Analyzer - API, CLI & Web App for analyzing & finding a person's profile across +350 social media \ websites. It includes different string analysis and detection modules, you can choose which combination of modules to use during the investigation process. +Social Analyzer - API, CLI & Web App for analyzing & finding a person's profile across +400 social media \ websites. It includes different string analysis and detection modules, you can choose which combination of modules to use during the investigation process. The detection modules utilize a rating mechanism based on different detection techniques, which produces a rate value that starts from 0 to 100 (No-Maybe-Yes). This module intended to have less false positive and it's documented in this [Wiki](https://github.com/qeeqbox/social-analyzer/wiki) link diff --git a/README.rst b/README.rst index 4185f3e..76ab33e 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,6 @@ .. image:: https://raw.githubusercontent.com/qeeqbox/social-analyzer/main/readme/socialanalyzerlogo_.png -Social-Analyzer - API, CLI & Web App for analyzing & finding a person's profile across +300 social media websites. It includes different string analysis and detection modules, you can choose which combination of modules to use during the investigation process. +Social-Analyzer - API, CLI & Web App for analyzing & finding a person's profile across +400 social media websites. It includes different string analysis and detection modules, you can choose which combination of modules to use during the investigation process. The detection modules utilize a rating mechanism based on different detection techniques, which produces a rate value that starts from 0 to 100 (No-Maybe-Yes). This module intended to have less false positive and it's documented in this `Wiki `_ link diff --git a/app.py b/app.py index 3844cf7..a418d8e 100644 --- a/app.py +++ b/app.py @@ -48,8 +48,8 @@ GENERIC_DETECTION = [] LOG = getLogger("social-analyzer") SITES_PATH = path.join(path.dirname(__file__), "data", "sites.json") LANGUAGES_PATH = path.join(path.dirname(__file__), "data", "languages.json") -STRINGS_PAGES = recompile('captcha-info|Please enable cookies', IGNORECASE) -STRINGS_TITLES = recompile('found|attention required|cloudflare', IGNORECASE) +STRINGS_PAGES = recompile('captcha-info|Please enable cookies|Completing the CAPTCHA', IGNORECASE) +STRINGS_TITLES = recompile('not found|blocked|attention required|cloudflare', IGNORECASE) LANGUAGES_JSON = {} WORKERS = 15 CUSTOM_MESSAGE = 51 @@ -60,8 +60,8 @@ with open(LANGUAGES_PATH) as f: def delete_keys(in_object, keys): - ''' - delete specific keys from object + ''' + delete specific keys from object ''' for key in keys: @@ -71,7 +71,7 @@ def delete_keys(in_object, keys): def clean_up_item(in_object, keys_str): - ''' + ''' delete specific keys from object (user input) ''' @@ -89,7 +89,7 @@ def clean_up_item(in_object, keys_str): def get_language_by_guessing(text): - ''' + ''' guess language by text, this needs long text ''' @@ -101,7 +101,7 @@ def get_language_by_guessing(text): def get_language_by_parsing(source): - ''' + ''' guess language by parsing the lang tag ''' @@ -113,7 +113,7 @@ def get_language_by_parsing(source): def check_errors(on_off=None): - ''' + ''' wrapper function for debugging ''' @@ -133,12 +133,12 @@ def check_errors(on_off=None): class CustomHandler(Handler): - ''' + ''' custom stream handler ''' def __init__(self, argv=None): - ''' + ''' int, user choices needed ''' @@ -146,7 +146,7 @@ class CustomHandler(Handler): self.argv = argv def emit(self, record): - ''' + ''' emit, based on user choices ''' @@ -178,7 +178,7 @@ class CustomHandler(Handler): @check_errors(True) def setup_logger(uuid=None, file=False, argv=None): - ''' + ''' setup a logger for logs in the temp folder ''' @@ -196,7 +196,7 @@ def setup_logger(uuid=None, file=False, argv=None): @check_errors(True) def init_detections(detections): - ''' + ''' load websites_entries, shared_detections and generic_detection ''' @@ -209,7 +209,7 @@ def init_detections(detections): def get_website(site): - ''' + ''' extract domain from website ''' @@ -219,7 +219,7 @@ def get_website(site): def list_all_websites(): - ''' + ''' list all the available websites' entries ''' @@ -232,14 +232,14 @@ def list_all_websites(): @check_errors(True) def find_username_normal(req): - ''' + ''' main find usernames logic using ThreadPoolExecutor ''' resutls = [] def fetch_url(site, username, options): - ''' + ''' this runs for every website entry ''' @@ -279,7 +279,7 @@ def find_username_normal(req): detections_count = 0 def check_url(url): - ''' + ''' check if url is okay ''' @@ -290,7 +290,7 @@ def find_username_normal(req): return False def merge_dicts(temp_dict): - ''' + ''' ''' result = {} @@ -303,7 +303,7 @@ def find_username_normal(req): return result def detect_logic(detections): - ''' + ''' check for detections in website entry ''' @@ -336,7 +336,7 @@ def find_username_normal(req): return temp_profile, temp_detected, detections_count def detect(): - ''' + ''' main detect logic ''' @@ -417,7 +417,9 @@ def find_username_normal(req): temp_profile["title"] = "filtered" if research(STRINGS_PAGES, temp_profile["text"]): temp_profile["text"] = "filtered" + temp_profile["title"] = "filtered" if research(STRINGS_TITLES, temp_profile["title"]): + temp_profile["text"] = "filtered" temp_profile["title"] = "filtered" with suppress(Exception): @@ -517,7 +519,7 @@ def find_username_normal(req): @check_errors(True) def check_user_cli(argv): - ''' + ''' main cli logic ''' @@ -616,7 +618,7 @@ def check_user_cli(argv): def msg(): - ''' + ''' welcome message ''' diff --git a/data/sites.json b/data/sites.json index 38a54fd..46e171e 100644 --- a/data/sites.json +++ b/data/sites.json @@ -346,6 +346,26 @@ "extract": [], "type": "Internet" }, + { + "url": "https://app.intigriti.com/profile/{username}", + "detections": [ + { + "return": "true", + "string": "user-name", + "type": "normal" + }, + { + "return": "true", + "string": "profile/{username}", + "type": "normal" + } + ], + "selected": "false", + "timeout": 0, + "implicit": 0, + "extract": [], + "type": "Business" + }, { "url": "https://archive.org/details/@{username}", "detections": [ @@ -689,6 +709,26 @@ "extract": [], "type": "Internet" }, + { + "url": "https://bit.dev/{username}", + "detections": [ + { + "return": "true", + "string": "\">{username}", + "type": "normal" + }, + { + "return": "true", + "string": "FOLLOWERS", + "type": "normal" + } + ], + "selected": "false", + "timeout": 0, + "implicit": 0, + "extract": [], + "type": "Computers" + }, { "url": "https://bitbucket.org/{username}", "detections": [ @@ -953,6 +993,26 @@ "extract": [], "type": "Internet" }, + { + "url": "https://carousell.sg/{username}", + "detections": [ + { + "return": "true", + "string": "photos/profiles", + "type": "normal" + }, + { + "return": "true", + "string": "@{username}", + "type": "normal" + } + ], + "selected": "false", + "timeout": 0, + "implicit": 0, + "extract": [], + "type": "Videos" + }, { "url": "https://carswap.me/{username}", "detections": [ @@ -1268,6 +1328,26 @@ "extract": [], "type": "Computers" }, + { + "url": "https://coderwall.com/{username}", + "detections": [ + { + "return": "true", + "string": "content=\"/{username}\"", + "type": "normal" + }, + { + "return": "true", + "string": "overflow-hidden", + "type": "normal" + } + ], + "selected": "false", + "timeout": 0, + "implicit": 0, + "extract": [], + "type": "Computers" + }, { "url": "https://codewars.com/users/{username}", "detections": [ @@ -1511,6 +1591,31 @@ "extract": [], "type": "Internet" }, + { + "url": "https://cplusplus.com/user/{username}/", + "detections": [ + { + "return": "false", + "string": "Not Found", + "type": "normal" + }, + { + "return": "true", + "string": "/user/{username}/", + "type": "normal" + }, + { + "return": "true", + "string": "I_content", + "type": "normal" + } + ], + "selected": "false", + "timeout": 0, + "implicit": 0, + "extract": [], + "type": "Computers" + }, { "url": "https://cracked.com/members/{username}", "detections": [ @@ -1841,6 +1946,26 @@ "extract": [], "type": "Computers" }, + { + "url": "https://digitalocean.com/community/users/{username}", + "detections": [ + { + "return": "true", + "string": "\"description\"", + "type": "normal" + }, + { + "return": "true", + "string": "users/{username}", + "type": "normal" + } + ], + "selected": "false", + "timeout": 0, + "implicit": 0, + "extract": [], + "type": "Jobs" + }, { "url": "https://discogs.com/user/{username}", "detections": [ @@ -2281,6 +2406,26 @@ "extract": [], "type": "Games" }, + { + "url": "https://erome.com/{username}", + "detections": [ + { + "return": "true", + "string": "user-profile", + "type": "normal" + }, + { + "return": "true", + "string": "user-info", + "type": "normal" + } + ], + "selected": "false", + "timeout": 0, + "implicit": 0, + "extract": [], + "type": "Adult" + }, { "url": "https://escapistmagazine.com/profiles/view/{username}", "detections": [ @@ -2456,6 +2601,26 @@ "extract": [], "type": "Internet" }, + { + "url": "https://f6s.com/{username}", + "detections": [ + { + "return": "true", + "string": "profile-picture", + "type": "normal" + }, + { + "return": "true", + "string": "profile-details", + "type": "normal" + } + ], + "selected": "false", + "timeout": 0, + "implicit": 0, + "extract": [], + "type": "Business" + }, { "url": "https://facebook.com/{username}", "detections": [ @@ -3662,6 +3827,26 @@ "extract": [], "type": "Internet" }, + { + "url": "https://guru.com/freelancers/{username}", + "detections": [ + { + "return": "true", + "string": "profile__", + "type": "normal" + }, + { + "return": "true", + "string": "freelancer-details", + "type": "normal" + } + ], + "selected": "false", + "timeout": 0, + "implicit": 0, + "extract": [], + "type": "Jobs" + }, { "url": "https://gurushots.com/{username}/photos", "detections": [], @@ -3716,6 +3901,26 @@ "extract": [], "type": "Internet" }, + { + "url": "https://hackerearth.com/@{username}", + "detections": [ + { + "return": "true", + "string": "profile-card", + "type": "normal" + }, + { + "return": "true", + "string": "profile-detail", + "type": "normal" + } + ], + "selected": "false", + "timeout": 0, + "implicit": 0, + "extract": [], + "type": "Computers" + }, { "url": "https://hackernoon.com/u/{username}", "detections": [ @@ -4073,6 +4278,26 @@ "extract": [], "type": "Internet" }, + { + "url": "https://ifunny.co/user/{username}", + "detections": [ + { + "return": "true", + "string": "user/{username}", + "type": "normal" + }, + { + "return": "true", + "string": "\"nick\":\"{username}\"", + "type": "normal" + } + ], + "selected": "false", + "timeout": 0, + "implicit": 0, + "extract": [], + "type": "Entertainment" + }, { "url": "https://imagefap.com/profile/{username}", "detections": [ @@ -4727,6 +4952,26 @@ "extract": [], "type": "Computers" }, + { + "url": "https://legiit.com/{username}", + "detections": [ + { + "return": "true", + "string": "user-avatar", + "type": "normal" + }, + { + "return": "true", + "string": "category-box", + "type": "normal" + } + ], + "selected": "false", + "timeout": 0, + "implicit": 0, + "extract": [], + "type": "Shopping" + }, { "url": "https://letterboxd.com/{username}", "detections": [ @@ -5097,6 +5342,26 @@ "extract": [], "type": "Internet" }, + { + "url": "https://metacafe.com/channels/{username}/", + "detections": [ + { + "return": "true", + "string": "og:url", + "type": "normal" + }, + { + "return": "true", + "string": "channels/{username}/", + "type": "normal" + } + ], + "selected": "false", + "timeout": 0, + "implicit": 0, + "extract": [], + "type": "Videos" + }, { "url": "https://metacritic.com/user/{username}", "detections": [ @@ -5252,6 +5517,31 @@ "extract": [], "type": "Internet" }, + { + "url": "https://muckrack.com/{username}", + "detections": [ + { + "return": "true", + "string": "profile-details", + "type": "normal" + }, + { + "return": "true", + "string": "profile-bio", + "type": "normal" + }, + { + "return": "true", + "string": "com/{username}", + "type": "normal" + } + ], + "selected": "false", + "timeout": 0, + "implicit": 0, + "extract": [], + "type": "Adult" + }, { "url": "https://munzee.com/m/{username}", "detections": [ @@ -5516,6 +5806,26 @@ "extract": [], "type": "Internet" }, + { + "url": "https://nimo.tv/{username}", + "detections": [ + { + "return": "true", + "string": "og:description", + "type": "normal" + }, + { + "return": "true", + "string": "\"alise\"", + "type": "normal" + } + ], + "selected": "false", + "timeout": 0, + "implicit": 0, + "extract": [], + "type": "Videos" + }, { "url": "https://ninjakiwi.com/profile/{username}", "detections": [ @@ -6677,6 +6987,26 @@ "extract": [], "type": "Computers" }, + { + "url": "https://republic.co/{username}", + "detections": [ + { + "return": "true", + "string": "user_profile", + "type": "normal" + }, + { + "return": "true", + "string": "\"{username}\"", + "type": "normal" + } + ], + "selected": "false", + "timeout": 0, + "implicit": 0, + "extract": [], + "type": "Internet" + }, { "url": "https://researchgate.net/profile/{username}", "detections": [ @@ -7635,6 +7965,26 @@ "extract": [], "type": "Internet" }, + { + "url": "https://trepup.com/{username}", + "detections": [ + { + "return": "true", + "string": "businessName", + "type": "normal" + }, + { + "return": "true", + "string": "BusinessAbout", + "type": "normal" + } + ], + "selected": "false", + "timeout": 0, + "implicit": 0, + "extract": [], + "type": "Business" + }, { "url": "https://tripadvisor.com/members/{username}", "detections": [ @@ -7685,6 +8035,26 @@ "extract": [], "type": "Computers" }, + { + "url": "https://tutsplus.com/authors/{username}", + "detections": [ + { + "return": "true", + "string": "profile__", + "type": "normal" + }, + { + "return": "true", + "string": "\"/authors/{username}", + "type": "normal" + } + ], + "selected": "false", + "timeout": 0, + "implicit": 0, + "extract": [], + "type": "Computers" + }, { "url": "https://twitch.tv/{username}", "detections": [ diff --git a/info b/info index 16eef02..426711f 100644 --- a/info +++ b/info @@ -2,8 +2,8 @@ "build":"pass", "test":"pass", "grid_test":"pass", - "websites":"368", - "detections":"946", + "websites":"402", + "detections":"1011", "special":"3", "linux":"pass", "windows":"pass", @@ -11,4 +11,4 @@ "full_scan":"15 workers < 23secs", "max_retries":"3", "awaiting_verification":"22", - "auto_testing":"4779f35e-c96f-4691-a56d-c119901082ee"} + "auto_testing":"ab595dd8-fc1b-416d-9a0f-0b06e3471812"} diff --git a/modules/fast-scan.js b/modules/fast-scan.js index c8674ae..add2e14 100644 --- a/modules/fast-scan.js +++ b/modules/fast-scan.js @@ -128,8 +128,15 @@ async function find_username_site(uuid, username, options, site) { helper.verbose && console.log(err); } - temp_profile.title = title; + temp_profile.text = temp_profile.text.replace(/(\r\n|\n|\r)/gm, ""); + temp_profile.title = title.replace(/(\r\n|\n|\r)/gm, ""); temp_profile.language = language; + + if (helper.strings_titles.test(temp_profile.title) || helper.strings_pages.test(temp_profile.text)){ + temp_profile.title = "filtered" + temp_profile.text = "filtered" + } + if (temp_profile.good == "true") { var temp_value = ((temp_profile["found"] / detections_count) * 100).toFixed(2) temp_profile.rate = "%" + temp_value; diff --git a/modules/helper.js b/modules/helper.js index 16cb7c0..dfdf0ea 100644 --- a/modules/helper.js +++ b/modules/helper.js @@ -75,6 +75,9 @@ var parsed_json = JSON.parse(fs.readFileSync(dict_json_path)); var logs_queue = Promise.resolve(); +var strings_pages = new RegExp('captcha-info|Please enable cookies|Completing the CAPTCHA', 'i') +var strings_titles = new RegExp('not found|blocked|attention required|cloudflare', 'i') + function get_log_file(uuid) { _uuid = uuid.replace(/[^a-zA-Z0-9\-]+/g, ''); _string = slash(path.join('logs', _uuid + "_log.txt")) @@ -302,6 +305,8 @@ async function setup_tecert() { } module.exports = { + strings_pages, + strings_titles, tecert_file, setup_tecert, compare_objects, diff --git a/modules/slow-scan.js b/modules/slow-scan.js index 9aa1db9..284c156 100644 --- a/modules/slow-scan.js +++ b/modules/slow-scan.js @@ -112,6 +112,15 @@ async function find_username_site(uuid, username, options, site) { temp_profile.text = sanitizeHtml(text_only); temp_profile.title = sanitizeHtml(title); temp_profile.language = language + + temp_profile.text = temp_profile.text.replace(/(\r\n|\n|\r)/gm, ""); + temp_profile.title = temp_profile.title.replace(/(\r\n|\n|\r)/gm, ""); + + if (helper.strings_titles.test(temp_profile.title) || helper.strings_pages.test(temp_profile.text)){ + temp_profile.title = "filtered" + temp_profile.text = "filtered" + } + if (temp_profile.good == "true") { var temp_value = ((temp_profile["found"] / detections_count) * 100).toFixed(2) temp_profile.rate = "%" + temp_value;