diff --git a/README.md b/README.md index 1502f4c..9862ca6 100644 --- a/README.md +++ b/README.md @@ -14,12 +14,10 @@ The results from the analysis could help in investigating profiles related to su Make sure to download this tool from GitHub - (ONLY) ⚠️ ## Updates -- (Request) changed workers to 5 (If your system hardware & internet connection is great, change this to higher value - I will be adding an option to change it in the next update) +- Added retry & failed logics profiles to Python CLI (This will show websites that failed to connect) - (Request) Added detection level to python & NodeJS cli - (Request) Added GetUserProfilesFast to python & NodeJS cli - (Request) Added GetUserProfilesFast (This option will bring the profile regardless if detected or not) -- Optimized slow scan & fast scan -- Added Windows support ## Security Testing @@ -56,6 +54,7 @@ Profile images **will not** be blurred. If you want them to be blurred, turn tha - Python CLI & NodeJS CLI (limited to FindUserProfilesFast option) - Grid option for faster checking (limited to docker-compose) - Dump logs to folder or terminal (prettified) +- Adjust finding\getting profile workers (default 15) - [Wiki](https://github.com/qeeqbox/social-analyzer/wiki) ## Special Detections diff --git a/app.py b/app.py index 2e18ecc..764dd52 100644 --- a/app.py +++ b/app.py @@ -41,7 +41,7 @@ LOG = getLogger("social-analyzer") SITES_PATH = path.join("data","sites.json") LANGUAGES_PATH = path.join("data","languages.json") LANGUAGES_JSON = {} -WORKERS = 5 +WORKERS = 15 with open(LANGUAGES_PATH) as f: LANGUAGES_JSON = load(f) @@ -53,9 +53,11 @@ def delete_keys(object,keys): return object def clean_up_item(object,keys_str): - del object["image"] + with suppress(Exception): + del object["image"] if keys_str == "" or keys_str == None: - del object["text"] + with suppress(Exception): + del object["text"] else: for key in object.copy(): if key not in keys_str: @@ -118,6 +120,11 @@ def init_websites(): temp_list.append(item) return temp_list +def get_website(site): + x = get_fld(site, fix_protocol=True) + x = x.replace(".{username}","").replace("{username}.","") + return x + def list_all_websites(): if len(PARSED_SITES) > 0: for site in PARSED_SITES: @@ -172,74 +179,101 @@ def find_username_normal(req): } - with suppress(Exception): + try: response = get(site["url"].replace("{username}", username), timeout=5, headers=headers, verify=False) source = response.text response.close() + text_only = "unavailable"; + title = "unavailable"; - text_only = "unavailable"; - title = "unavailable"; + for detection in site["detections"]: + temp_found = "false"; + if detection["type"] in detection_level[detection_level["current"]]["fast"] and source != "": + detections_count += 1 + if detection["string"].replace("{username}", username).lower() in source.lower(): + temp_found = "true" + if detection["return"] == temp_found: + temp_profile["found"] += 1 - for detection in site["detections"]: - temp_found = "false"; - if detection["type"] in detection_level[detection_level["current"]]["fast"] and source != "": - detections_count += 1 - if detection["string"].replace("{username}", username).lower() in source.lower(): - temp_found = "true" - if detection["return"] == temp_found: - temp_profile["found"] += 1 + if temp_profile["found"] >= detection_level[detection_level["current"]]["found"] and detections_count >= detection_level[detection_level["current"]]["count"]: + temp_profile["good"] = "true" - if temp_profile["found"] >= detection_level[detection_level["current"]]["found"] and detections_count >= detection_level[detection_level["current"]]["count"]: - temp_profile["good"] = "true" + with suppress(Exception): + soup = BeautifulSoup(source, "html.parser") + [tag.extract() for tag in soup(["head", "title","style", "script", "[document]"])] + temp_profile["text"] = soup.getText() + temp_profile["text"] = resub("\s\s+", " ", temp_profile["text"]) + temp_profile["text"] = temp_profile["text"].replace("\n", "").replace("\t", "").replace("\r", "").strip() + with suppress(Exception): + temp_profile["language"] = get_language_by_parsing(source) + if temp_profile["language"] == "unavailable": + temp_profile["language"] = get_language_by_guessing(temp_profile["text"]) + with suppress(Exception): + temp_profile["title"] = BeautifulSoup(source, "html.parser").title.string + temp_profile["title"] = resub("\s\s+", " ", temp_profile["title"]) + temp_profile["title"] = temp_profile["title"].replace("\n", "").replace("\t", "").replace("\r", "").strip() + if temp_profile["text"] == "": + temp_profile["text"] = "unavailable" + with suppress(Exception): + if detections_count != 0: + temp_profile["rate"] = "%" + str(round(((temp_profile["found"] / detections_count) * 100), 2)) - with suppress(Exception): - soup = BeautifulSoup(source, "html.parser") - [tag.extract() for tag in soup(["head", "title","style", "script", "[document]"])] - temp_profile["text"] = soup.getText() - temp_profile["text"] = resub("\s\s+", " ", temp_profile["text"]) - temp_profile["text"] = temp_profile["text"].replace("\n", "").replace("\t", "").replace("\r", "").strip() - with suppress(Exception): - temp_profile["language"] = get_language_by_parsing(source) - if temp_profile["language"] == "unavailable": - temp_profile["language"] = get_language_by_guessing(temp_profile["text"]) - with suppress(Exception): - temp_profile["title"] = BeautifulSoup(source, "html.parser").title.string - temp_profile["title"] = resub("\s\s+", " ", temp_profile["title"]) - temp_profile["title"] = temp_profile["title"].replace("\n", "").replace("\t", "").replace("\r", "").strip() - if temp_profile["text"] == "": - temp_profile["text"] = "unavailable" - if temp_profile["title"] == "": - temp_profile["title"] = "unavailable" - with suppress(Exception): - if detections_count != 0: - temp_profile["rate"] = "%" + str(round(((temp_profile["found"] / detections_count) * 100), 2)) + temp_profile["link"] = site["url"].replace("{username}", req["body"]["string"]); + temp_profile["type"] = site["type"] - temp_profile["link"] = site["url"].replace("{username}", req["body"]["string"]); - temp_profile["type"] = site["type"] + if "FindUserProfilesFast" in options and "GetUserProfilesFast" not in options: + temp_profile["method"] = "find" + elif "GetUserProfilesFast" in options and "FindUserProfilesFast" not in options: + temp_profile["method"] = "get" + elif "FindUserProfilesFast" in options and "GetUserProfilesFast" in options: + temp_profile["method"] = "all" - if "FindUserProfilesFast" in options and "GetUserProfilesFast" not in options: - temp_profile["method"] = "find" - elif "GetUserProfilesFast" in options and "FindUserProfilesFast" not in options: - temp_profile["method"] = "get" - elif "FindUserProfilesFast" in options and "GetUserProfilesFast" in options: - temp_profile["method"] = "all" + copy_temp_profile = temp_profile.copy() + return 1,site["url"], copy_temp_profile + except Exception as e: + pass - copy_temp_profile = temp_profile.copy() - return copy_temp_profile + return None,site["url"],[] + + for i in range(3): + if len(PARSED_SITES) > 0: + with ThreadPoolExecutor(max_workers=WORKERS) as executor: + future_fetch_url = (executor.submit(fetch_url, site, req["body"]["string"],req["body"]["options"]) for site in PARSED_SITES if site["selected"] == "true") + for future in as_completed(future_fetch_url): + try: + good, site, data = future.result() + if good: + PARSED_SITES[:] = [d for d in PARSED_SITES if d.get('url') != site] + resutls.append(data) + else: + LOG.info("[Waiting to retry] "+ get_website(site)) + except Exception as e: + pass + + + if len(PARSED_SITES) > 0: + for site in PARSED_SITES: + temp_profile = { + "found": 0, + "image": "", + "link": "", + "rate": "", + "title": "unavailable", + "language": "unavailable", + "text": "unavailable", + "type": "", + "good":"", + "method":"" + } + temp_profile["link"] = site["url"].replace("{username}", req["body"]["string"]); + temp_profile["type"] = site["type"] + resutls.append(temp_profile) - with ThreadPoolExecutor(max_workers=WORKERS) as executor: - future_fetch_url = (executor.submit(fetch_url, site, req["body"]["string"],req["body"]["options"]) for site in PARSED_SITES if site["selected"] == "true") - for future in as_completed(future_fetch_url): - try: - data = future.result() - resutls.append(future.result()) - except Exception as e: - pass return resutls @check_errors(True) def check_user_cli(argv): - temp_detected = {"detected":[],"unknown":[]} + temp_detected = {"detected":[],"unknown":[],"failed":[]} temp_keys = {"found": 0,"link": "","rate": "","title": "","text": ""}; temp_options = "GetUserProfilesFast,FindUserProfilesFast" if argv.method != "": @@ -282,6 +316,10 @@ def check_user_cli(argv): item = delete_keys(item,["found","rate","method","good"]) item = clean_up_item(item,argv.options) temp_detected["unknown"].append(item) + else: + item = delete_keys(item,["found","rate","method","good","text","title","language","rate"]) + item = clean_up_item(item,argv.options) + temp_detected["failed"].append(item) if len(temp_detected["detected"]) == 0: del temp_detected["detected"] @@ -289,6 +327,9 @@ def check_user_cli(argv): if len(temp_detected["unknown"]) == 0: del temp_detected["unknown"]; + if len(temp_detected["failed"]) == 0: + del temp_detected["failed"]; + if argv.output == "pretty" or argv.output == "": if 'detected' in temp_detected: LOG.info("\n[Detected] {} Profile[s]\n".format(len(temp_detected['detected']))); @@ -298,7 +339,11 @@ def check_user_cli(argv): LOG.info("\n[unknown] {} Profile[s]\n".format(len(temp_detected['unknown']))); for item in temp_detected['unknown']: LOG.info(highlight(dumps(item, sort_keys=True, indent=4), lexers.JsonLexer(), formatters.TerminalFormatter())) - + if 'failed' in temp_detected: + LOG.info("\n[failed] {} Profile[s]\n".format(len(temp_detected['failed']))); + for item in temp_detected['failed']: + LOG.info(highlight(dumps(item, sort_keys=True, indent=4), lexers.JsonLexer(), formatters.TerminalFormatter())) + if argv.output == "json": print(dumps(temp_detected, sort_keys=True, indent=None)) diff --git a/info b/info index e89a8e1..717ceec 100644 --- a/info +++ b/info @@ -1,4 +1,4 @@ -{"version":"2021.V.2.20", +{"version":"2021.V.2.21", "build":"pass", "test":"pass", "grid_test":"pass", @@ -9,5 +9,6 @@ "windows":"pass", "docker":"pass", "full_scan":"15 workers < 30secs", + "max retries":"3", "awaiting_verification":"22", - "auto_testing":"bce782fc-5add-4fbd-8036-dfb4c486cd19"} + "auto_testing":"f4807195-cd53-47a4-aed4-de6c9bee938e"}