mirror of
https://github.com/qeeqbox/social-analyzer.git
synced 2026-09-11 12:17:46 +02:00
[special request] added custom search queries - ref fb2d75a6
🕵️♀️
This commit is contained in:
G
@@ -3,20 +3,22 @@
|
||||
#
|
||||
[](https://github.com/qeeqbox/social-analyzer/blob/main/info) [](https://github.com/qeeqbox/social-analyzer/blob/main/info) [](https://github.com/qeeqbox/social-analyzer/blob/main/info) [](https://github.com/qeeqbox/social-analyzer/blob/main/info) [](https://github.com/qeeqbox/social-analyzer/blob/main/info) [](https://github.com/qeeqbox/social-analyzer/blob/main/info) [](https://github.com/qeeqbox/social-analyzer/blob/main/info) [](https://github.com/qeeqbox/social-analyzer/stargazers)
|
||||
|
||||
API and Web App for analyzing & finding a person profile across +300 social media websites. It includes different string analysis and detection modules, you can choose which combination of modules to use during the investigation. The detection modules utilize a rating mechanism based on different detection techniques, which produces a rate value that starts from 0 to 100 (No-Maybe-Yes)
|
||||
API and Web App for analyzing & finding a person profile across +300 social media websites. It includes different string analysis and detection modules, you can choose which combination of modules to use during the investigation.
|
||||
|
||||
This project could help in investigating profiles related to suspicious or malicious activities such as [Cyberbullying](https://en.wikipedia.org/wiki/Wikipedia:Cyberbullying), [Cybergrooming](https://de.wikipedia.org/wiki/Cyber-Grooming), [Cyberstalking](https://en.wikipedia.org/wiki/Cyberstalking) & spreading [Misinformation](https://en.wikipedia.org/wiki/Misinformation) etc..
|
||||
The detection modules utilize a rating mechanism based on different detection techniques, which produces a rate value that starts from 0 to 100 (No-Maybe-Yes)
|
||||
|
||||
This project could help in investigating profiles related to suspicious or malicious activities such as [cyberbullying](https://en.wikipedia.org/wiki/Wikipedia:Cyberbullying), [cybergrooming](https://de.wikipedia.org/wiki/Cyber-Grooming), [cyberstalking](https://en.wikipedia.org/wiki/Cyberstalking) & [spreading misinformation](https://en.wikipedia.org/wiki/Misinformation) etc..
|
||||
|
||||
**Please submit your contribution in a Pull Request!**
|
||||
|
||||
## Updates
|
||||
- Added Custom Search Query
|
||||
- Added similarity checking to FindOrigins
|
||||
- Added Proxy option for FindUserProfilesFast & FindUserProfilesSlow
|
||||
- Added documentation on how to write detections [Wiki](https://github.com/qeeqbox/social-analyzer/wiki) 👏👏👏
|
||||
- Added Proxy option for FindUserProfilesFast & FindUserProfilesSlow 👏
|
||||
- Added documentation on how to write detections [Wiki](https://github.com/qeeqbox/social-analyzer/wiki) 👏
|
||||
- Added detection type:advanced (This allows detection on websites that manipulate DOM) 👏
|
||||
- Added more automated tests for handling Pull Requests (will be adding documentation on how PR are being handled) 👏
|
||||
- Added FindOrigins (This extracts well-known names with their origins from the username) 👏
|
||||
- Enhanceced FindUserProfilesSlow & ShowUserProfilesSlow checking speed (Limited to docker-compose) 👏
|
||||
|
||||
## Security Testing
|
||||
|
||||
@@ -44,19 +46,22 @@ Profile images **will not** be blurred. If you want them to be blurred, turn tha
|
||||
## Features
|
||||
- String Analysis
|
||||
- Search Engine Lookup
|
||||
- Multi Layers detections
|
||||
- Multi Layers Detections
|
||||
- Most Common Names & Words
|
||||
- Convert Numbers to Letters
|
||||
- Find Profile Normal (Fast)
|
||||
- Find Profile Advanced (Slow)
|
||||
- Find Profile Special (Slow)
|
||||
- Profile Screenshot (Slow)
|
||||
- Python CLI (Limited to FindUserProfilesFast option)
|
||||
- NodeJS CLI (Limited to FindUserProfilesFast option)
|
||||
- Optional timeout and implicit wait for each detection (Some websites have a delay logic implemented in the backend)
|
||||
- Adding descriptions' logic to websites (will be adding the categories later on)
|
||||
- Find Name Origins
|
||||
- Custom Search Queries
|
||||
- Websites description
|
||||
- Custom user-agent option
|
||||
- Dump Logs to folder
|
||||
- Optional timeout & implicit wait for each detection
|
||||
- Python CLI (Limited to FindUserProfilesFast option)
|
||||
- NodeJS CLI (Limited to FindUserProfilesFast option)
|
||||
- Grid option (Limited to docker-compose) 👏
|
||||
|
||||
## Special Detections
|
||||
- Facebook
|
||||
|
||||
@@ -113,7 +113,7 @@ function log_to_file_queue(uuid, msg) {
|
||||
});
|
||||
}
|
||||
|
||||
async function get_url_wrapper_json(url, time) {
|
||||
async function get_url_wrapper_json(url, time=5) {
|
||||
try {
|
||||
let http_promise = new Promise((resolve, reject) => {
|
||||
var request = https.get(url, header_options, function(res) {
|
||||
@@ -141,8 +141,8 @@ async function get_url_wrapper_json(url, time) {
|
||||
});
|
||||
let response_body = await http_promise;
|
||||
return response_body
|
||||
} catch (error) {
|
||||
verbose && console.log('');
|
||||
} catch (err) {
|
||||
verbose && console.log(err);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -181,7 +181,7 @@ async function find_username_special(req) {
|
||||
}
|
||||
});
|
||||
const results = await async.parallelLimit(functions, 5);
|
||||
console.log(`Total time ${new Date() - time}`);
|
||||
verbose && console.log(`Total time ${new Date() - time}`);
|
||||
return results.filter(item => item !== undefined)
|
||||
}
|
||||
|
||||
@@ -265,7 +265,7 @@ async function find_username_advanced(req) {
|
||||
}
|
||||
});
|
||||
const results = await async.parallelLimit(functions, 8);
|
||||
console.log(`Total time ${new Date() - time}`);
|
||||
verbose && console.log(`Total time ${new Date() - time}`);
|
||||
return results.filter(item => item !== undefined)
|
||||
}
|
||||
|
||||
@@ -310,7 +310,7 @@ async function find_username_site_new(uuid, username, options, site) {
|
||||
script: timeout
|
||||
};
|
||||
|
||||
console.log(timeouts)
|
||||
verbose && console.log(timeouts)
|
||||
|
||||
var source = "";
|
||||
var data = "";
|
||||
@@ -634,6 +634,45 @@ async function find_username_advanced_2(username, options) {
|
||||
return results.filter(item => item !== undefined);
|
||||
}
|
||||
|
||||
async function custom_search_ouputs(req) {
|
||||
var possible_parameters = ['user', 'profile', 'account']
|
||||
const time = new Date();
|
||||
const functions = [];
|
||||
possible_parameters.forEach((key) => {
|
||||
functions.push(custom_search_ouputs_website.bind(null, req.body.uuid, req.body.string, key));
|
||||
});
|
||||
const results = await async.parallelLimit(functions, 6);
|
||||
verbose && console.log(`Total time ${new Date() - time}`);
|
||||
var merged = [].concat.apply([], results.filter(item => item !== undefined));
|
||||
return merged
|
||||
}
|
||||
|
||||
async function custom_search_ouputs_website(uuid, name, key) {
|
||||
|
||||
return new Promise(async (resolve, reject) => {
|
||||
try {
|
||||
var results = []
|
||||
log_to_file_queue(uuid, "[Custom Search Using] " + key)
|
||||
var url = "https://www.googleapis.com/customsearch/v1?key={0}&cx={1}&q={2}:{3}".replace("{0}", google_api_key).replace("{1}", google_api_cs).replace("{2}", key).replace("{3}", name);
|
||||
var response = await get_url_wrapper_json(url);
|
||||
if (response.data != '') {
|
||||
if ('items' in response.data){
|
||||
response.data.items.forEach((key) => {
|
||||
results.push({site:get_site_from_url(key.link),link:key.link,snippet:key.snippet})
|
||||
});
|
||||
}
|
||||
}
|
||||
if (results.length > 0) {
|
||||
resolve(results);
|
||||
} else {
|
||||
resolve(undefined)
|
||||
}
|
||||
} catch (err) {
|
||||
resolve(undefined)
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
app.get("/get_settings", async function(req, res, next) {
|
||||
temp_list = [];
|
||||
temp_list = await Promise.all(parsed_sites.map(async (site, index) => {
|
||||
@@ -1077,6 +1116,7 @@ app.post("/url", async function(req, res, next) {
|
||||
var names_origins = []
|
||||
var words_info = []
|
||||
var temp_words = []
|
||||
var custom_search = []
|
||||
if (req.body.string == null || req.body.string == "") {
|
||||
res.json("Error");
|
||||
} else {
|
||||
@@ -1106,6 +1146,11 @@ app.post("/url", async function(req, res, next) {
|
||||
await check_engines(req, info);
|
||||
log_to_file_queue(req.body.uuid, "[Done] Lookup")
|
||||
}
|
||||
if (req.body.option.includes("CustomSearch")) {
|
||||
log_to_file_queue(req.body.uuid, "[Starting] Custom Search")
|
||||
custom_search = await custom_search_ouputs(req);
|
||||
log_to_file_queue(req.body.uuid, "[Done] Custom Search")
|
||||
}
|
||||
if (req.body.option.includes("SplitWordsByUpperCase")) {
|
||||
try {
|
||||
req.body.string.match(/[A-Z][a-z]+/g).forEach((item) => {
|
||||
@@ -1233,7 +1278,8 @@ app.post("/url", async function(req, res, next) {
|
||||
user_info_normal: user_info_normal,
|
||||
user_info_advanced: user_info_advanced,
|
||||
user_info_special: user_info_special,
|
||||
names_origins: names_origins
|
||||
names_origins: names_origins,
|
||||
custom_search:custom_search
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
+32
-4
@@ -228,6 +228,7 @@
|
||||
#output-section-4,
|
||||
#most-common-words-section,
|
||||
#words-info-section,
|
||||
#custom-search-section,
|
||||
#names-origins-section {
|
||||
display: none;
|
||||
}
|
||||
@@ -264,6 +265,14 @@
|
||||
word-break: break-word;
|
||||
}
|
||||
|
||||
#custom-search-section table tr th:first-child,
|
||||
#custom-search-section table tr td:first-child {
|
||||
width: 150px;
|
||||
min-width: 150px;
|
||||
max-width: 150px;
|
||||
word-break: break-word;
|
||||
}
|
||||
|
||||
.box-border-style {
|
||||
background-color: #2b2c2c !important;
|
||||
-webkit-border-radius: 2px !important;
|
||||
@@ -670,16 +679,17 @@
|
||||
<option value="LookUps">LookUps</option>
|
||||
<option value="WordInfo">WordInfo</option>
|
||||
<option value="MostCommon">MostCommon</option>
|
||||
<option value="SplitWordsByUpperCase">SplitWordsByUpperCase</option>
|
||||
<option value="SplitWordsByAlphabet">SplitWordsByAlphabet</option>
|
||||
<option value="ConvertNumbers">ConvertNumbers</option>
|
||||
<option value="FindOrigins">FindOrigins</option>
|
||||
<option value="FindNumbers">FindNumbers</option>
|
||||
<option value="FindSymbols">FindSymbols</option>
|
||||
<option value="CustomSearch">CustomSearch</option>
|
||||
<option value="ConvertNumbers">ConvertNumbers</option>
|
||||
<option value="SplitWordsByAlphabet">SplitWordsByAlphabet</option>
|
||||
<option value="SplitWordsByUpperCase">SplitWordsByUpperCase</option>
|
||||
<option value="FindUserProfilesFast">FindUserProfilesFast</option>
|
||||
<option value="FindUserProfilesSlow">FindUserProfilesSlow</option>
|
||||
<option value="ShowUserProfilesSlow">ShowUserProfilesSlow</option>
|
||||
<option value="FindUserProfilesSpecial">FindUserProfilesSpecial</option>
|
||||
<option value="FindOrigins">FindOrigins</option>
|
||||
</select>
|
||||
<div class="cell-separator"></div>
|
||||
<input type="button" id="analyze-normal" value="Analyze" />
|
||||
@@ -724,6 +734,14 @@
|
||||
<div id="words-info-tables">
|
||||
</div>
|
||||
</div>
|
||||
<div class="box-border-style" id="custom-search-section">
|
||||
<div class="div-header">
|
||||
<div class="div-left">Custom Search</div>
|
||||
<div class="div-right"></div>
|
||||
</div>
|
||||
<div id="custom-search-table">
|
||||
</div>
|
||||
</div>
|
||||
<div class="box-border-style" id="possible-words-section">
|
||||
<div class="div-header">
|
||||
<div class="div-left">Possible words for combinations</div>
|
||||
@@ -832,6 +850,8 @@
|
||||
$('#output-table-4').html("");
|
||||
$('#names-origins-section').hide();
|
||||
$('#names-origins-table').html("");
|
||||
$('#custom-search-section').hide();
|
||||
$('#custom-search-table').html("");
|
||||
$('#logs').html("");
|
||||
clearInterval(interval_logs);
|
||||
}
|
||||
@@ -1011,6 +1031,14 @@
|
||||
$("#names-origins-table").last().append('<table><tr><th>origin</th><th>name</th><th>matched</th><th>similar</th><th>gender</th></tr>' + temp_tr + '</table>')
|
||||
$('#names-origins-section').show();
|
||||
}
|
||||
if (data.custom_search.length > 0) {
|
||||
var temp_tr = ""
|
||||
Object.keys(data.custom_search).forEach(function(item) {
|
||||
temp_tr += '<tr><td>' + data.custom_search[item].site + '</td><td><a href="' + data.custom_search[item].link + '">' + data.custom_search[item].link + '</a></td><td>' + data.custom_search[item].snippet + '</td></tr>'
|
||||
});
|
||||
$("#custom-search-table").last().append('<table><tr><th>site</th><th>link</th><th>snippet</th></tr>' + temp_tr + '</table>')
|
||||
$('#custom-search-section').show();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user