WEBDIENSTE/social-analyzer
1
0
Fork 0
mirror of https://github.com/qeeqbox/social-analyzer.git synced 2026-08-17 19:25:42 +02:00
Code Issues Packages Projects Releases Wiki Activity

[auto-b] added more websites - ref 37cce582

Browse Source
This commit is contained in:
G
2021-03-22 14:12:39 -07:00
committed by GitHub
parent f95867c8b4
commit 8b24e5dffc
8 changed files with 422 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+1 -1
View File
@@ -3,7 +3,7 @@
#
[![Generic badge](https://img.shields.io/badge/dynamic/json.svg?url=https://raw.githubusercontent.com/qeeqbox/social-analyzer/main/info&label=version&query=$.version&colorB=blue&style=flat-square)](https://github.com/qeeqbox/social-analyzer/blob/main/info) [![Generic badge](https://img.shields.io/badge/dynamic/json.svg?url=https://raw.githubusercontent.com/qeeqbox/social-analyzer/main/info&label=verified%20sites&query=$.websites&colorB=blue&style=flat-square)](https://github.com/qeeqbox/social-analyzer/blob/main/info) [![Generic badge](https://img.shields.io/badge/dynamic/json.svg?url=https://raw.githubusercontent.com/qeeqbox/social-analyzer/main/info&label=verified%20detections&query=$.detections&colorB=blue&style=flat-square)](https://github.com/qeeqbox/social-analyzer/blob/main/info) [![Generic badge](https://img.shields.io/badge/dynamic/json.svg?url=https://raw.githubusercontent.com/qeeqbox/social-analyzer/main/info&label=build&query=$.build&colorB=green&style=flat-square)](https://github.com/qeeqbox/social-analyzer/blob/main/info) [![Generic badge](https://img.shields.io/badge/dynamic/json.svg?url=https://raw.githubusercontent.com/qeeqbox/social-analyzer/main/info&label=test&query=$.test&colorB=green&style=flat-square)](https://github.com/qeeqbox/social-analyzer/blob/main/info) [![Generic badge](https://img.shields.io/badge/dynamic/json.svg?url=https://raw.githubusercontent.com/qeeqbox/social-analyzer/main/info&label=docker&query=$.docker&colorB=green&style=flat-square)](https://github.com/qeeqbox/social-analyzer/blob/main/info) [![Generic badge](https://img.shields.io/static/v1?label=%F0%9F%91%8D&message=!&color=yellow&style=flat-square)](https://github.com/qeeqbox/social-analyzer/stargazers)
Social Analyzer - API, CLI & Web App for analyzing & finding a person's profile across +350 social media \ websites. It includes different string analysis and detection modules, you can choose which combination of modules to use during the investigation process.
Social Analyzer - API, CLI & Web App for analyzing & finding a person's profile across +400 social media \ websites. It includes different string analysis and detection modules, you can choose which combination of modules to use during the investigation process.
The detection modules utilize a rating mechanism based on different detection techniques, which produces a rate value that starts from 0 to 100 (No-Maybe-Yes). This module intended to have less false positive and it's documented in this [Wiki](https://github.com/qeeqbox/social-analyzer/wiki) link
README.rst
+1 -1
View File
@@ -1,6 +1,6 @@
.. image:: https://raw.githubusercontent.com/qeeqbox/social-analyzer/main/readme/socialanalyzerlogo_.png
Social-Analyzer - API, CLI & Web App for analyzing & finding a person's profile across +300 social media websites. It includes different string analysis and detection modules, you can choose which combination of modules to use during the investigation process.
Social-Analyzer - API, CLI & Web App for analyzing & finding a person's profile across +400 social media websites. It includes different string analysis and detection modules, you can choose which combination of modules to use during the investigation process.
The detection modules utilize a rating mechanism based on different detection techniques, which produces a rate value that starts from 0 to 100 (No-Maybe-Yes). This module intended to have less false positive and it's documented in this `Wiki <https://github.com/qeeqbox/social-analyzer/wiki>`_ link
app.py
+25 -23
View File
@@ -48,8 +48,8 @@ GENERIC_DETECTION = []
LOG = getLogger("social-analyzer")
SITES_PATH = path.join(path.dirname(__file__), "data", "sites.json")
LANGUAGES_PATH = path.join(path.dirname(__file__), "data", "languages.json")
STRINGS_PAGES = recompile('captcha-info|Please enable cookies', IGNORECASE)
STRINGS_TITLES = recompile('found|attention required|cloudflare', IGNORECASE)
STRINGS_PAGES = recompile('captcha-info|Please enable cookies|Completing the CAPTCHA', IGNORECASE)
STRINGS_TITLES = recompile('not found|blocked|attention required|cloudflare', IGNORECASE)
LANGUAGES_JSON = {}
WORKERS = 15
CUSTOM_MESSAGE = 51
@@ -60,8 +60,8 @@ with open(LANGUAGES_PATH) as f:
def delete_keys(in_object, keys):
'''
delete specific keys from object
'''
delete specific keys from object
'''
for key in keys:
@@ -71,7 +71,7 @@ def delete_keys(in_object, keys):
def clean_up_item(in_object, keys_str):
'''
'''
delete specific keys from object (user input)
'''
@@ -89,7 +89,7 @@ def clean_up_item(in_object, keys_str):
def get_language_by_guessing(text):
'''
'''
guess language by text, this needs long text
'''
@@ -101,7 +101,7 @@ def get_language_by_guessing(text):
def get_language_by_parsing(source):
'''
'''
guess language by parsing the lang tag
'''
@@ -113,7 +113,7 @@ def get_language_by_parsing(source):
def check_errors(on_off=None):
'''
'''
wrapper function for debugging
'''
@@ -133,12 +133,12 @@ def check_errors(on_off=None):
class CustomHandler(Handler):
'''
'''
custom stream handler
'''
def __init__(self, argv=None):
'''
'''
int, user choices needed
'''
@@ -146,7 +146,7 @@ class CustomHandler(Handler):
self.argv = argv
def emit(self, record):
'''
'''
emit, based on user choices
'''
@@ -178,7 +178,7 @@ class CustomHandler(Handler):
@check_errors(True)
def setup_logger(uuid=None, file=False, argv=None):
'''
'''
setup a logger for logs in the temp folder
'''
@@ -196,7 +196,7 @@ def setup_logger(uuid=None, file=False, argv=None):
@check_errors(True)
def init_detections(detections):
'''
'''
load websites_entries, shared_detections and generic_detection
'''
@@ -209,7 +209,7 @@ def init_detections(detections):
def get_website(site):
'''
'''
extract domain from website
'''
@@ -219,7 +219,7 @@ def get_website(site):
def list_all_websites():
'''
'''
list all the available websites' entries
'''
@@ -232,14 +232,14 @@ def list_all_websites():
@check_errors(True)
def find_username_normal(req):
'''
'''
main find usernames logic using ThreadPoolExecutor
'''
resutls = []
def fetch_url(site, username, options):
'''
'''
this runs for every website entry
'''
@@ -279,7 +279,7 @@ def find_username_normal(req):
detections_count = 0
def check_url(url):
'''
'''
check if url is okay
'''
@@ -290,7 +290,7 @@ def find_username_normal(req):
return False
def merge_dicts(temp_dict):
'''
'''
'''
result = {}
@@ -303,7 +303,7 @@ def find_username_normal(req):
return result
def detect_logic(detections):
'''
'''
check for detections in website entry
'''
@@ -336,7 +336,7 @@ def find_username_normal(req):
return temp_profile, temp_detected, detections_count
def detect():
'''
'''
main detect logic
'''
@@ -417,7 +417,9 @@ def find_username_normal(req):
temp_profile["title"] = "filtered"
if research(STRINGS_PAGES, temp_profile["text"]):
temp_profile["text"] = "filtered"
temp_profile["title"] = "filtered"
if research(STRINGS_TITLES, temp_profile["title"]):
temp_profile["text"] = "filtered"
temp_profile["title"] = "filtered"
with suppress(Exception):
@@ -517,7 +519,7 @@ def find_username_normal(req):
@check_errors(True)
def check_user_cli(argv):
'''
'''
main cli logic
'''
@@ -616,7 +618,7 @@ def check_user_cli(argv):
def msg():
'''
'''
welcome message
'''
data/sites.json
+370
View File
@@ -346,6 +346,26 @@
"extract": [],
"type": "Internet"
},
{
"url": "https://app.intigriti.com/profile/{username}",
"detections": [
{
"return": "true",
"string": "user-name",
"type": "normal"
},
{
"return": "true",
"string": "profile/{username}",
"type": "normal"
}
],
"selected": "false",
"timeout": 0,
"implicit": 0,
"extract": [],
"type": "Business"
},
{
"url": "https://archive.org/details/@{username}",
"detections": [
@@ -689,6 +709,26 @@
"extract": [],
"type": "Internet"
},
{
"url": "https://bit.dev/{username}",
"detections": [
{
"return": "true",
"string": "\">{username}",
"type": "normal"
},
{
"return": "true",
"string": "FOLLOWERS",
"type": "normal"
}
],
"selected": "false",
"timeout": 0,
"implicit": 0,
"extract": [],
"type": "Computers"
},
{
"url": "https://bitbucket.org/{username}",
"detections": [
@@ -953,6 +993,26 @@
"extract": [],
"type": "Internet"
},
{
"url": "https://carousell.sg/{username}",
"detections": [
{
"return": "true",
"string": "photos/profiles",
"type": "normal"
},
{
"return": "true",
"string": "@{username}",
"type": "normal"
}
],
"selected": "false",
"timeout": 0,
"implicit": 0,
"extract": [],
"type": "Videos"
},
{
"url": "https://carswap.me/{username}",
"detections": [
@@ -1268,6 +1328,26 @@
"extract": [],
"type": "Computers"
},
{
"url": "https://coderwall.com/{username}",
"detections": [
{
"return": "true",
"string": "content=\"/{username}\"",
"type": "normal"
},
{
"return": "true",
"string": "overflow-hidden",
"type": "normal"
}
],
"selected": "false",
"timeout": 0,
"implicit": 0,
"extract": [],
"type": "Computers"
},
{
"url": "https://codewars.com/users/{username}",
"detections": [
@@ -1511,6 +1591,31 @@
"extract": [],
"type": "Internet"
},
{
"url": "https://cplusplus.com/user/{username}/",
"detections": [
{
"return": "false",
"string": "Not Found",
"type": "normal"
},
{
"return": "true",
"string": "/user/{username}/",
"type": "normal"
},
{
"return": "true",
"string": "I_content",
"type": "normal"
}
],
"selected": "false",
"timeout": 0,
"implicit": 0,
"extract": [],
"type": "Computers"
},
{
"url": "https://cracked.com/members/{username}",
"detections": [
@@ -1841,6 +1946,26 @@
"extract": [],
"type": "Computers"
},
{
"url": "https://digitalocean.com/community/users/{username}",
"detections": [
{
"return": "true",
"string": "\"description\"",
"type": "normal"
},
{
"return": "true",
"string": "users/{username}",
"type": "normal"
}
],
"selected": "false",
"timeout": 0,
"implicit": 0,
"extract": [],
"type": "Jobs"
},
{
"url": "https://discogs.com/user/{username}",
"detections": [
@@ -2281,6 +2406,26 @@
"extract": [],
"type": "Games"
},
{
"url": "https://erome.com/{username}",
"detections": [
{
"return": "true",
"string": "user-profile",
"type": "normal"
},
{
"return": "true",
"string": "user-info",
"type": "normal"
}
],
"selected": "false",
"timeout": 0,
"implicit": 0,
"extract": [],
"type": "Adult"
},
{
"url": "https://escapistmagazine.com/profiles/view/{username}",
"detections": [
@@ -2456,6 +2601,26 @@
"extract": [],
"type": "Internet"
},
{
"url": "https://f6s.com/{username}",
"detections": [
{
"return": "true",
"string": "profile-picture",
"type": "normal"
},
{
"return": "true",
"string": "profile-details",
"type": "normal"
}
],
"selected": "false",
"timeout": 0,
"implicit": 0,
"extract": [],
"type": "Business"
},
{
"url": "https://facebook.com/{username}",
"detections": [
@@ -3662,6 +3827,26 @@
"extract": [],
"type": "Internet"
},
{
"url": "https://guru.com/freelancers/{username}",
"detections": [
{
"return": "true",
"string": "profile__",
"type": "normal"
},
{
"return": "true",
"string": "freelancer-details",
"type": "normal"
}
],
"selected": "false",
"timeout": 0,
"implicit": 0,
"extract": [],
"type": "Jobs"
},
{
"url": "https://gurushots.com/{username}/photos",
"detections": [],
@@ -3716,6 +3901,26 @@
"extract": [],
"type": "Internet"
},
{
"url": "https://hackerearth.com/@{username}",
"detections": [
{
"return": "true",
"string": "profile-card",
"type": "normal"
},
{
"return": "true",
"string": "profile-detail",
"type": "normal"
}
],
"selected": "false",
"timeout": 0,
"implicit": 0,
"extract": [],
"type": "Computers"
},
{
"url": "https://hackernoon.com/u/{username}",
"detections": [
@@ -4073,6 +4278,26 @@
"extract": [],
"type": "Internet"
},
{
"url": "https://ifunny.co/user/{username}",
"detections": [
{
"return": "true",
"string": "user/{username}",
"type": "normal"
},
{
"return": "true",
"string": "\"nick\":\"{username}\"",
"type": "normal"
}
],
"selected": "false",
"timeout": 0,
"implicit": 0,
"extract": [],
"type": "Entertainment"
},
{
"url": "https://imagefap.com/profile/{username}",
"detections": [
@@ -4727,6 +4952,26 @@
"extract": [],
"type": "Computers"
},
{
"url": "https://legiit.com/{username}",
"detections": [
{
"return": "true",
"string": "user-avatar",
"type": "normal"
},
{
"return": "true",
"string": "category-box",
"type": "normal"
}
],
"selected": "false",
"timeout": 0,
"implicit": 0,
"extract": [],
"type": "Shopping"
},
{
"url": "https://letterboxd.com/{username}",
"detections": [
@@ -5097,6 +5342,26 @@
"extract": [],
"type": "Internet"
},
{
"url": "https://metacafe.com/channels/{username}/",
"detections": [
{
"return": "true",
"string": "og:url",
"type": "normal"
},
{
"return": "true",
"string": "channels/{username}/",
"type": "normal"
}
],
"selected": "false",
"timeout": 0,
"implicit": 0,
"extract": [],
"type": "Videos"
},
{
"url": "https://metacritic.com/user/{username}",
"detections": [
@@ -5252,6 +5517,31 @@
"extract": [],
"type": "Internet"
},
{
"url": "https://muckrack.com/{username}",
"detections": [
{
"return": "true",
"string": "profile-details",
"type": "normal"
},
{
"return": "true",
"string": "profile-bio",
"type": "normal"
},
{
"return": "true",
"string": "com/{username}",
"type": "normal"
}
],
"selected": "false",
"timeout": 0,
"implicit": 0,
"extract": [],
"type": "Adult"
},
{
"url": "https://munzee.com/m/{username}",
"detections": [
@@ -5516,6 +5806,26 @@
"extract": [],
"type": "Internet"
},
{
"url": "https://nimo.tv/{username}",
"detections": [
{
"return": "true",
"string": "og:description",
"type": "normal"
},
{
"return": "true",
"string": "\"alise\"",
"type": "normal"
}
],
"selected": "false",
"timeout": 0,
"implicit": 0,
"extract": [],
"type": "Videos"
},
{
"url": "https://ninjakiwi.com/profile/{username}",
"detections": [
@@ -6677,6 +6987,26 @@
"extract": [],
"type": "Computers"
},
{
"url": "https://republic.co/{username}",
"detections": [
{
"return": "true",
"string": "user_profile",
"type": "normal"
},
{
"return": "true",
"string": "\"{username}\"",
"type": "normal"
}
],
"selected": "false",
"timeout": 0,
"implicit": 0,
"extract": [],
"type": "Internet"
},
{
"url": "https://researchgate.net/profile/{username}",
"detections": [
@@ -7635,6 +7965,26 @@
"extract": [],
"type": "Internet"
},
{
"url": "https://trepup.com/{username}",
"detections": [
{
"return": "true",
"string": "businessName",
"type": "normal"
},
{
"return": "true",
"string": "BusinessAbout",
"type": "normal"
}
],
"selected": "false",
"timeout": 0,
"implicit": 0,
"extract": [],
"type": "Business"
},
{
"url": "https://tripadvisor.com/members/{username}",
"detections": [
@@ -7685,6 +8035,26 @@
"extract": [],
"type": "Computers"
},
{
"url": "https://tutsplus.com/authors/{username}",
"detections": [
{
"return": "true",
"string": "profile__",
"type": "normal"
},
{
"return": "true",
"string": "\"/authors/{username}",
"type": "normal"
}
],
"selected": "false",
"timeout": 0,
"implicit": 0,
"extract": [],
"type": "Computers"
},
{
"url": "https://twitch.tv/{username}",
"detections": [
info
+3 -3
View File
@@ -2,8 +2,8 @@
"build":"pass",
"test":"pass",
"grid_test":"pass",
"websites":"368",
"detections":"946",
"websites":"402",
"detections":"1011",
"special":"3",
"linux":"pass",
"windows":"pass",
@@ -11,4 +11,4 @@
"full_scan":"15 workers < 23secs",
"max_retries":"3",
"awaiting_verification":"22",
"auto_testing":"4779f35e-c96f-4691-a56d-c119901082ee"}
"auto_testing":"ab595dd8-fc1b-416d-9a0f-0b06e3471812"}
modules/fast-scan.js
+8 -1
View File
@@ -128,8 +128,15 @@ async function find_username_site(uuid, username, options, site) {
helper.verbose && console.log(err);
}
temp_profile.title = title;
temp_profile.text = temp_profile.text.replace(/(\r\n|\n|\r)/gm, "");
temp_profile.title = title.replace(/(\r\n|\n|\r)/gm, "");
temp_profile.language = language;
if (helper.strings_titles.test(temp_profile.title) || helper.strings_pages.test(temp_profile.text)){
temp_profile.title = "filtered"
temp_profile.text = "filtered"
}
if (temp_profile.good == "true") {
var temp_value = ((temp_profile["found"] / detections_count) * 100).toFixed(2)
temp_profile.rate = "%" + temp_value;
modules/helper.js
+5
View File
@@ -75,6 +75,9 @@ var parsed_json = JSON.parse(fs.readFileSync(dict_json_path));
var logs_queue = Promise.resolve();
var strings_pages = new RegExp('captcha-info|Please enable cookies|Completing the CAPTCHA', 'i')
var strings_titles = new RegExp('not found|blocked|attention required|cloudflare', 'i')
function get_log_file(uuid) {
_uuid = uuid.replace(/[^a-zA-Z0-9\-]+/g, '');
_string = slash(path.join('logs', _uuid + "_log.txt"))
@@ -302,6 +305,8 @@ async function setup_tecert() {
}
module.exports = {
strings_pages,
strings_titles,
tecert_file,
setup_tecert,
compare_objects,
modules/slow-scan.js
+9
View File
@@ -112,6 +112,15 @@ async function find_username_site(uuid, username, options, site) {
temp_profile.text = sanitizeHtml(text_only);
temp_profile.title = sanitizeHtml(title);
temp_profile.language = language
temp_profile.text = temp_profile.text.replace(/(\r\n|\n|\r)/gm, "");
temp_profile.title = temp_profile.title.replace(/(\r\n|\n|\r)/gm, "");
if (helper.strings_titles.test(temp_profile.title) || helper.strings_pages.test(temp_profile.text)){
temp_profile.title = "filtered"
temp_profile.text = "filtered"
}
if (temp_profile.good == "true") {
var temp_value = ((temp_profile["found"] / detections_count) * 100).toFixed(2)
temp_profile.rate = "%" + temp_value;