mirror of
https://github.com/qeeqbox/social-analyzer.git
synced 2026-09-08 10:47:45 +02:00
[auto-b] Extract metadata & patterns available on the web app - ref 01b7489b
This commit is contained in:
G
@@ -148,5 +148,5 @@ docker build -t social-analyzer . && docker run -p 9005:9005 -it social-analyzer
|
||||
## Disclaimer\Notes
|
||||
- Make sure to download this tool from GitHub
|
||||
- This is a security project (Treat it as a security project)
|
||||
- This tool meant to be used locally (It does not have any type of Access Control)
|
||||
- If you want your website to be excluded from this project list, please reach out to me
|
||||
- This tool meant to be used locally not as a service (It does not have any type of Access Control)
|
||||
|
||||
@@ -426,12 +426,17 @@ async function check_user_cli(argv) {
|
||||
var temp_options = "GetUserProfilesFast,FindUserProfilesFast"
|
||||
if (argv.method != "") {
|
||||
if (argv.method == "find") {
|
||||
temp_options = "FindUserProfilesFast"
|
||||
temp_options = ",FindUserProfilesFast,"
|
||||
} else if (argv.method == "get") {
|
||||
temp_options = "GetUserProfilesFast"
|
||||
temp_options = ",GetUserProfilesFast,"
|
||||
}
|
||||
}
|
||||
|
||||
if (argv.extract){
|
||||
temp_options += ",ExtractPatterns,"
|
||||
}
|
||||
if (argv.metadata){
|
||||
temp_options += ",ExtractMetadata,"
|
||||
}
|
||||
var req = {
|
||||
'body': {
|
||||
'uuid': random_string,
|
||||
@@ -475,7 +480,7 @@ async function check_user_cli(argv) {
|
||||
item = clean_up_item(item,argv.options)
|
||||
temp_detected.detected.push(item)
|
||||
} else {
|
||||
item = delete_keys(item,['found','rate','status','method','good'])
|
||||
item = delete_keys(item,['found','rate','status','method','good','extracted','metadata'])
|
||||
item = clean_up_item(item,argv.options)
|
||||
temp_detected.unknown.push(item)
|
||||
}
|
||||
@@ -486,11 +491,11 @@ async function check_user_cli(argv) {
|
||||
temp_detected.detected.push(item)
|
||||
}
|
||||
} else if (item.method == "get") {
|
||||
item = delete_keys(item,['found','rate','status','method','good'])
|
||||
item = delete_keys(item,['found','rate','status','method','good','extracted','metadata'])
|
||||
item = clean_up_item(item,argv.options)
|
||||
temp_detected.unknown.push(item)
|
||||
} else if (item.method == "failed") {
|
||||
item = delete_keys(item,['found','rate','status','method','good','text','language','title','type'])
|
||||
item = delete_keys(item,['found','rate','status','method','good','text','language','title','type','extracted','metadata'])
|
||||
item = clean_up_item(item,argv.options)
|
||||
temp_detected.failed.push(item)
|
||||
}
|
||||
|
||||
+19
-17
@@ -25,7 +25,9 @@ async function find_username_normal(req) {
|
||||
}
|
||||
|
||||
all_results = all_results.filter(item => item)
|
||||
return all_results.sort((first, second) => {return helper.compare_objects(first, second, 'rate')})
|
||||
return all_results.sort((first, second) => {
|
||||
return helper.compare_objects(first, second, 'rate')
|
||||
})
|
||||
}
|
||||
|
||||
async function get_failed(req, failed) {
|
||||
@@ -131,13 +133,11 @@ async function find_username_site(uuid, username, options, site) {
|
||||
if (temp_profile.good == "true") {
|
||||
var temp_value = ((temp_profile["found"] / detections_count) * 100).toFixed(2)
|
||||
temp_profile.rate = "%" + temp_value;
|
||||
if (temp_value >= 100.00){
|
||||
if (temp_value >= 100.00) {
|
||||
temp_profile.status = "good"
|
||||
}
|
||||
else if (temp_value >= 50.00 && temp_value < 100.00){
|
||||
} else if (temp_value >= 50.00 && temp_value < 100.00) {
|
||||
temp_profile.status = "maybe"
|
||||
}
|
||||
else{
|
||||
} else {
|
||||
temp_profile.status = "bad"
|
||||
}
|
||||
}
|
||||
@@ -145,18 +145,20 @@ async function find_username_site(uuid, username, options, site) {
|
||||
temp_profile.link = site.url.replace("{username}", username);
|
||||
temp_profile.type = site.type
|
||||
|
||||
if (temp_profile.status == "good"){
|
||||
var temp_metadata_list = []
|
||||
temp_metadata_list = await extraction.extract_metadata(site,source)
|
||||
if (temp_metadata_list.length > 0)
|
||||
{
|
||||
temp_profile.metadata = temp_metadata_list
|
||||
if (temp_profile.status == "good") {
|
||||
if (options.includes("ExtractPatterns")) {
|
||||
var temp_extracted_list = []
|
||||
temp_extracted_list = await extraction.extract_patterns(site, source)
|
||||
if (temp_extracted_list.length > 0) {
|
||||
temp_profile.extracted = temp_extracted_list
|
||||
}
|
||||
}
|
||||
var temp_extracted_list = []
|
||||
temp_extracted_list = await extraction.extract_patterns(site,source)
|
||||
if (temp_extracted_list.length > 0)
|
||||
{
|
||||
temp_profile.extracted = temp_extracted_list
|
||||
if (options.includes("ExtractMetadata")) {
|
||||
var temp_metadata_list = []
|
||||
temp_metadata_list = await extraction.extract_metadata(site, source)
|
||||
if (temp_metadata_list.length > 0) {
|
||||
temp_profile.metadata = temp_metadata_list
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+20
-1
@@ -1,4 +1,5 @@
|
||||
var helper = require('./helper.js')
|
||||
var helper = require('./helper.js');
|
||||
var extraction = require('./extraction.js')
|
||||
var async = require("async");
|
||||
var sanitizeHtml = require("sanitize-html");
|
||||
var sanitizeHtml = require("sanitize-html");
|
||||
@@ -122,6 +123,24 @@ async function find_username_site(uuid, username, options, site) {
|
||||
temp_profile.status = "bad"
|
||||
}
|
||||
}
|
||||
|
||||
if (temp_profile.status == "good") {
|
||||
if (options.includes("ExtractPatterns")) {
|
||||
var temp_extracted_list = []
|
||||
temp_extracted_list = await extraction.extract_patterns(site, source)
|
||||
if (temp_extracted_list.length > 0) {
|
||||
temp_profile.extracted = temp_extracted_list
|
||||
}
|
||||
}
|
||||
if (options.includes("ExtractMetadata")) {
|
||||
var temp_metadata_list = []
|
||||
temp_metadata_list = await extraction.extract_metadata(site, source)
|
||||
if (temp_metadata_list.length > 0) {
|
||||
temp_profile.metadata = temp_metadata_list
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
temp_profile.link = site.url.replace("{username}", username);
|
||||
temp_profile.type = site.type
|
||||
resolve(temp_profile);
|
||||
|
||||
+101
-58
@@ -219,11 +219,11 @@
|
||||
padding-right: 2px;
|
||||
}
|
||||
|
||||
.selectize-dropdown .optgroup{
|
||||
.selectize-dropdown .optgroup {
|
||||
border-top: 1px solid #666666;
|
||||
}
|
||||
|
||||
.optgroup-header{
|
||||
.optgroup-header {
|
||||
border: none !important;
|
||||
background-color: #383939 !important;
|
||||
color: #d0d0d0 !important;
|
||||
@@ -603,17 +603,16 @@
|
||||
opacity: 0.1 !important;
|
||||
}
|
||||
|
||||
#div-right-icon{
|
||||
float:right;
|
||||
#div-right-icon {
|
||||
float: right;
|
||||
font-size: 16px;
|
||||
}
|
||||
|
||||
#logs-section pre{
|
||||
#logs-section pre {
|
||||
color: #d0d0d0 !important;
|
||||
font-size: 12px !important;
|
||||
font-family: monospace !important;
|
||||
}
|
||||
|
||||
</style>
|
||||
|
||||
<body>
|
||||
@@ -708,26 +707,30 @@
|
||||
<div class="cell-separator"></div>
|
||||
<select name="analyzed-options" placeholder="Choose some options..." id="analyzed-options" multiple="multiple">
|
||||
<optgroup label="Name Analysis">
|
||||
<option value="WordInfo">WordInfo</option>
|
||||
<option value="MostCommon">MostCommon</option>
|
||||
<option value="FindOrigins">FindOrigins</option>
|
||||
<option value="WordInfo">WordInfo</option>
|
||||
<option value="MostCommon">MostCommon</option>
|
||||
<option value="FindOrigins">FindOrigins</option>
|
||||
</optgroup>
|
||||
<optgroup label="Find Profiles">
|
||||
<option value="LookUps">LookUps</option>
|
||||
<option value="CustomSearch">CustomSearch</option>
|
||||
<option value="FindUserProfilesFast,GetUserProfilesFast">FindUserProfilesFast</option>
|
||||
<option value="FindUserProfilesSlow">FindUserProfilesSlow</option>
|
||||
<option value="FindUserProfilesSpecial">FindUserProfilesSpecial</option>
|
||||
<option value="LookUps">LookUps</option>
|
||||
<option value="CustomSearch">CustomSearch</option>
|
||||
<option value="FindUserProfilesFast,GetUserProfilesFast">FindUserProfilesFast</option>
|
||||
<option value="FindUserProfilesSlow">FindUserProfilesSlow</option>
|
||||
<option value="FindUserProfilesSpecial">FindUserProfilesSpecial</option>
|
||||
</optgroup>
|
||||
<optgroup label="Show Profiles">
|
||||
<option value="ShowUserProfilesSlow">ShowUserProfilesSlow</option>
|
||||
<option value="ShowUserProfilesSlow">ShowUserProfilesSlow</option>
|
||||
</optgroup>
|
||||
<optgroup label="Extract">
|
||||
<option value="ExtractPatterns">ExtractPatterns</option>
|
||||
<option value="ExtractMetadata">ExtractMetadata</option>
|
||||
</optgroup>
|
||||
<optgroup label="Other options">
|
||||
<option value="FindNumbers">FindNumbers</option>
|
||||
<option value="FindSymbols">FindSymbols</option>
|
||||
<option value="ConvertNumbers">ConvertNumbers</option>
|
||||
<option value="SplitWordsByAlphabet">SplitWordsByAlphabet</option>
|
||||
<option value="SplitWordsByUpperCase">SplitWordsByUpperCase</option>
|
||||
<option value="FindNumbers">FindNumbers</option>
|
||||
<option value="FindSymbols">FindSymbols</option>
|
||||
<option value="ConvertNumbers">ConvertNumbers</option>
|
||||
<option value="SplitWordsByAlphabet">SplitWordsByAlphabet</option>
|
||||
<option value="SplitWordsByUpperCase">SplitWordsByUpperCase</option>
|
||||
</optgroup>
|
||||
</select>
|
||||
<div class="cell-separator"></div>
|
||||
@@ -839,8 +842,11 @@
|
||||
</div>
|
||||
</div>
|
||||
<script>
|
||||
|
||||
random_names = ["sophia","liam","riley","jackson","emma","aiden","ava","elijah","isabella","grayson","aria","lucas","aaliyah","oliver","amelia","caden","mia","mateo","layla","muhammad","zoe","mason","camilla","carter","charlotte","jayden","eliana","ethan","mila","sebastian","everly","james","luna","michael","avery","benjamin","evelyn","logan","harper","leo","lily","luca","ella","alexander","gianna","levi","chloe","daniel","adalyn","josiah","charlie","henry","isla","jayce","ellie","julian","leah","jack","nora","ryan","scarlett","jacob","maya","asher","abigail","wyatt","madison","william","aubrey","owen","emily","gabriel","kinsley","miles","elena","lincoln","paisley","ezra","madelyn","isaiah","aurora","luke","peyton","cameron","nova","caleb","emilia","isaac","hannah","carson","sarah","samuel","ariana","colton","penelope","maverick","lila","matthew"]
|
||||
random_names = ["sophia", "liam", "riley", "jackson", "emma", "aiden", "ava", "elijah", "isabella", "grayson", "aria", "lucas", "aaliyah", "oliver", "amelia", "caden", "mia", "mateo", "layla", "muhammad", "zoe", "mason", "camilla", "carter",
|
||||
"charlotte", "jayden", "eliana", "ethan", "mila", "sebastian", "everly", "james", "luna", "michael", "avery", "benjamin", "evelyn", "logan", "harper", "leo", "lily", "luca", "ella", "alexander", "gianna", "levi", "chloe", "daniel", "adalyn",
|
||||
"josiah", "charlie", "henry", "isla", "jayce", "ellie", "julian", "leah", "jack", "nora", "ryan", "scarlett", "jacob", "maya", "asher", "abigail", "wyatt", "madison", "william", "aubrey", "owen", "emily", "gabriel", "kinsley", "miles",
|
||||
"elena", "lincoln", "paisley", "ezra", "madelyn", "isaiah", "aurora", "luke", "peyton", "cameron", "nova", "caleb", "emilia", "isaac", "hannah", "carson", "sarah", "samuel", "ariana", "colton", "penelope", "maverick", "lila", "matthew"
|
||||
]
|
||||
|
||||
//$("#string-to-analyze").attr("placeholder", random_names[(Math.random() * random_names.length) | 0]);
|
||||
$("#string-to-analyze").attr("placeholder", "johndoe");
|
||||
@@ -929,6 +935,41 @@
|
||||
});
|
||||
}
|
||||
|
||||
function add_extract(site) {
|
||||
var temp_extracted_all = '<div>Extracted: ' + "unavailable" + '</div>'
|
||||
var temp_metadata_all = '<div>Metadata: ' + "unavailable" + '</div>'
|
||||
try {
|
||||
if (site.extracted != "unavailable" && site.extracted != "") {
|
||||
temp_extracted_all = ""
|
||||
site.extracted.forEach(function(extracted, i) {
|
||||
var temp_extracted = ""
|
||||
Object.keys(extracted).forEach(function(key) {
|
||||
temp_extracted += key + ' : ' + extracted[key] + ' '
|
||||
});
|
||||
temp_extracted_all += '<div>Extracted ' + i + ': ' + temp_extracted + '</div>'
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
temp_extracted_all = '<div>Extracted: ' + "unavailable" + '</div>'
|
||||
}
|
||||
try {
|
||||
if (site.metadata != "unavailable" && site.metadata != "") {
|
||||
temp_metadata_all = ""
|
||||
site.metadata.forEach(function(extracted, i) {
|
||||
var temp_metadata = ""
|
||||
Object.keys(extracted).forEach(function(key) {
|
||||
temp_metadata += key + ' : ' + extracted[key] + ' '
|
||||
});
|
||||
temp_metadata_all += '<div>Metadata ' + i + ': ' + temp_metadata + '</div>'
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
temp_metadata_all = '<div>Metadata: ' + "unavailable" + '</div>'
|
||||
}
|
||||
|
||||
return temp_extracted_all + temp_metadata_all
|
||||
}
|
||||
|
||||
function add_table(items) {
|
||||
|
||||
var temp_th = ""
|
||||
@@ -1019,8 +1060,10 @@
|
||||
if (site.image != "") {
|
||||
temp_image = '<img src=' + site.image + '>'
|
||||
}
|
||||
temp_tr += '<div class="user-info-container"><div class="user-info-container-left"><div>Link: <a href="' + site.link + '">' + site.link + '</a></div><div>Rate: ' + site.rate + '</div><div>Status: ' + site.status + '</div><div>Title: ' + site.title +
|
||||
'</div><div>Language: ' + site.language + '</div><div>Description: ' + site.type + '</div><div>Text: ' + site.text + '</div></div><div class="user-info-container-right">' + temp_image + '</div></div>'
|
||||
var temp_extract = add_extract(site);
|
||||
temp_tr += '<div class="user-info-container"><div class="user-info-container-left"><div>Link: <a href="' + site.link + '">' + site.link + '</a></div><div>Rate: ' + site.rate + '</div><div>Status: ' + site.status +
|
||||
'</div><div>Title: ' + site.title +
|
||||
'</div><div>Language: ' + site.language + '</div><div>Description: ' + site.type + '</div><div>Text: ' + site.text + '</div>' + temp_extract + '</div><div class="user-info-container-right">' + temp_image + '</div></div>'
|
||||
}
|
||||
});
|
||||
|
||||
@@ -1047,27 +1090,30 @@
|
||||
|
||||
data.user_info_normal.data.forEach(function(site) {
|
||||
if (site.method == "all") {
|
||||
if (site.good == "true"){
|
||||
temp_tr += '<div class="user-info-container"><div class="user-info-container-left"><div>Link: <a href="' + site.link + '">' + site.link + '</a></div><div>Rate: ' + site.rate + '</div><div>Status: ' + site.status + '</div><div>Title: ' + site.title +
|
||||
'</div><div>Language: ' + site.language + '</div><div>Description: ' + site.type + '</div><div>Text: ' + site.text + '</div></div><div class="user-info-container-right"></div></div>'
|
||||
}
|
||||
else{
|
||||
var temp_extract = ""
|
||||
if (site.good == "true") {
|
||||
var temp_extract = add_extract(site);
|
||||
temp_tr += '<div class="user-info-container"><div class="user-info-container-left"><div>Link: <a href="' + site.link + '">' + site.link + '</a></div><div>Rate: ' + site.rate + '</div><div>Status: ' + site.status +
|
||||
'</div><div>Title: ' + site.title +
|
||||
'</div><div>Language: ' + site.language + '</div><div>Description: ' + site.type + '</div><div>Text: ' + site.text + '</div>' + temp_extract + '</div><div class="user-info-container-right"></div></div>'
|
||||
} else {
|
||||
temp_tr += '<div class="user-info-container user-info-container-color-unknown"><div class="user-info-container-left"><div>Link: <a href="' + site.link + '">' + site.link + '</a></div><div>Title: ' + site.title +
|
||||
'</div><div>Language: ' + site.language + '</div><div>Description: ' + site.type + '</div><div>Text: ' + site.text + '</div></div><div class="user-info-container-right"></div></div>'
|
||||
}
|
||||
}
|
||||
else if (site.method == "find") {
|
||||
if(site.good == "true"){
|
||||
temp_tr += '<div class="user-info-container"><div class="user-info-container-left"><div>Link: <a href="' + site.link + '">' + site.link + '</a></div><div>Rate: ' + site.rate + '</div><div>Status: ' + site.status + '</div><div>Title: ' + site.title +
|
||||
'</div><div>Language: ' + site.language + '</div><div>Description: ' + site.type + '</div><div>Text: ' + site.text + '</div></div><div class="user-info-container-right"></div></div>'
|
||||
} else if (site.method == "find") {
|
||||
if (site.good == "true") {
|
||||
var temp_extract = add_extract(site);
|
||||
temp_tr += '<div class="user-info-container"><div class="user-info-container-left"><div>Link: <a href="' + site.link + '">' + site.link + '</a></div><div>Rate: ' + site.rate + '</div><div>Status: ' + site.status +
|
||||
'</div><div>Title: ' + site.title +
|
||||
'</div><div>Language: ' + site.language + '</div><div>Description: ' + site.type + '</div><div>Text: ' + site.text + '</div>' + temp_extract + '</div><div class="user-info-container-right"></div></div>'
|
||||
}
|
||||
}
|
||||
else if (site.method == "get") {
|
||||
temp_tr += '<div class="user-info-container user-info-container-color-unknown"><div class="user-info-container-left"><div>Link: <a href="' + site.link + '">' + site.link + '</a></div><div>Rate: ' + site.rate + '</div><div>Status: ' + site.status + '</div><div>Title: ' + site.title +
|
||||
'</div><div>Language: ' + site.language + '</div><div>Description: ' + site.type + '</div><div>Text: ' + site.text + '</div></div><div class="user-info-container-right"></div></div>'
|
||||
}
|
||||
else if (site.method == "failed") {
|
||||
temp_tr += '<div class="user-info-container user-info-container-color-failed"><div class="user-info-container-left"><div>Link: <a href="' + site.link + '">' + site.link + '</a></div></div><div class="user-info-container-right"></div></div>'
|
||||
} else if (site.method == "get") {
|
||||
temp_tr += '<div class="user-info-container user-info-container-color-unknown"><div class="user-info-container-left"><div>Link: <a href="' + site.link + '">' + site.link + '</a></div><div>Rate: ' + site.rate +
|
||||
'</div><div>Status: ' + site.status + '</div><div>Title: ' + site.title +
|
||||
'</div><div>Language: ' + site.language + '</div><div>Description: ' + site.type + '</div><div>Text: ' + site.text + '</div></div><div class="user-info-container-right"></div></div>'
|
||||
} else if (site.method == "failed") {
|
||||
temp_tr += '<div class="user-info-container user-info-container-color-failed"><div class="user-info-container-left"><div>Link: <a href="' + site.link + '">' + site.link +
|
||||
'</a></div></div><div class="user-info-container-right"></div></div>'
|
||||
}
|
||||
});
|
||||
|
||||
@@ -1080,7 +1126,8 @@
|
||||
data.user_info_special.data.forEach(function(site) {
|
||||
if (site.found > 0) {
|
||||
temp_image = ""
|
||||
temp_tr += '<div class="user-info-container"><div class="user-info-container-left"><div>Link: <a href="' + site.link + '">' + site.link + '</a></div><div>Rate: ' + site.rate + '</div><div>Status: ' + site.status + '</div><div>Title: ' + site.title +
|
||||
temp_tr += '<div class="user-info-container"><div class="user-info-container-left"><div>Link: <a href="' + site.link + '">' + site.link + '</a></div><div>Rate: ' + site.rate + '</div><div>Status: ' + site.status +
|
||||
'</div><div>Title: ' + site.title +
|
||||
'</div><div>Description: ' + site.type + '</div><div>Text: ' + site.text + '</div></div><div class="user-info-container-right">' + temp_image + '</div></div>'
|
||||
}
|
||||
});
|
||||
@@ -1201,23 +1248,19 @@
|
||||
});
|
||||
|
||||
$(function() {
|
||||
$('.div-header').click(function() {
|
||||
if (this.innerText != "Enter Profile Name"){
|
||||
if ($(this).next().hasClass('show'))
|
||||
{
|
||||
$(this).find('#div-right-icon').removeClass("fa-caret-down")
|
||||
$(this).find('#div-right-icon').addClass("fa-caret-right")
|
||||
}
|
||||
else
|
||||
{
|
||||
$(this).find('#div-right-icon').removeClass("fa-caret-right")
|
||||
$(this).find('#div-right-icon').addClass("fa-caret-down")
|
||||
}
|
||||
$(this).next().collapse('toggle');
|
||||
}
|
||||
});
|
||||
$('.div-header').click(function() {
|
||||
if (this.innerText != "Enter Profile Name") {
|
||||
if ($(this).next().hasClass('show')) {
|
||||
$(this).find('#div-right-icon').removeClass("fa-caret-down")
|
||||
$(this).find('#div-right-icon').addClass("fa-caret-right")
|
||||
} else {
|
||||
$(this).find('#div-right-icon').removeClass("fa-caret-right")
|
||||
$(this).find('#div-right-icon').addClass("fa-caret-down")
|
||||
}
|
||||
$(this).next().collapse('toggle');
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
</script>
|
||||
</body>
|
||||
|
||||
|
||||
Reference in New Issue
Block a user