WEBDIENSTE/social-analyzer
1
0
Fork 0
mirror of https://github.com/qeeqbox/social-analyzer.git synced 2026-09-12 04:37:45 +02:00
Code Issues Packages Projects Releases Wiki Activity

[auto] new refactored version 2021.V.2.10 - ref 15c8fdb4

Browse Source
This commit is contained in:
G
2021-01-11 00:22:25 -08:00
committed by GitHub
parent efc13558a9
commit ad840860f3
13 changed files with 1299 additions and 1203 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+11 -11
View File
@@ -5,18 +5,18 @@
API and Web App for analyzing & finding a person profile across +300 social media websites. It includes different string analysis and detection modules, you can choose which combination of modules to use during the investigation.
The detection modules utilize a rating mechanism based on different detection techniques, which produces a rate value that starts from 0 to 100 (No-Maybe-Yes)
The detection modules utilize a rating mechanism based on different detection techniques, which produces a rate value that starts from 0 to 100 (No-Maybe-Yes). This model produces less *False Positive* unlike other projects
This project could help in investigating profiles related to suspicious or malicious activities such as [cyberbullying](https://en.wikipedia.org/wiki/Wikipedia:Cyberbullying), [cybergrooming](https://de.wikipedia.org/wiki/Cyber-Grooming) & [cyberstalking](https://en.wikipedia.org/wiki/Cyberstalking), [spreading misinformation](https://en.wikipedia.org/wiki/Misinformation). And, *"It's currently used by some law enforcement agencies in countries where resources are limited"*
**Pull Requests are welcomed!**
## Updates
- (Testing) A new refactored version (Coming soon!!)
- Added special detection for google 👏
- Added more customization to the CLI python version (It's a lot easier to integrate using the JSON output switch) 👏
- Added Custom Search Query 👏
- Added similarity checking to FindOrigins 👏
- Added a new refactored version 👏👏👏
- Added special detection for google
- Added more customization to the CLI python version (It's a lot easier to integrate using the JSON output switch)
- Added Custom Search Query
- Added similarity checking to FindOrigins
## Security Testing
@@ -52,11 +52,11 @@ Profile images **will not** be blurred. If you want them to be blurred, turn tha
- Custom user-agent, proxy, timeout & implicit wait
- Python CLI & NodeJS CLI (Limited to FindUserProfilesFast option)
- Grid Option for Faster Checking (Limited to docker-compose) 👏
- Dump Logs to folder or terminal
- Dump Logs to folder or terminal (prettified)
- [Wiki](https://github.com/qeeqbox/social-analyzer/wiki)
## Special Detections
- Facebook
- Facebook, @gmail and google
## Install and run as web app (NodeJS + NPM + Firefox)
```bash
@@ -81,9 +81,9 @@ cd social-analyzer
rm -rf package-lock.json node_modules
npm install lodash
npm install
# If you want to list all websites use node app.js -c -l
# If you want to list all websites use node app.js --cli --list
# Remember the following runs as FindUserProfilesFast
node app.js -c -m "fast" -u "username" -w "youtube pinterest tumblr"
node app.j --cli --mode "fast" --username "johndoe" --websites "youtube pinterest tumblr" --output "pretty"
```
## Install and run as CLI (Python3 + NPM + Firefox)
@@ -94,7 +94,7 @@ sudo apt-get install -y firefox-esr tesseract-ocr git
git clone https://github.com/qeeqbox/social-analyzer.git
cd social-analyzer
pip3 install lxml BeautifulSoup4 tld pygments
# If you want to list all websites use python3 app.py -c -l
# If you want to list all websites use python3 app.py --cli --list
# Remember the following runs as FindUserProfilesFast
python3 app.py --cli --mode "fast" --username "johndoe" --websites "youtube pinterest tumblr" --output "pretty"
```
app.js
+163 -1181
View File
File diff suppressed because it is too large Load Diff
info
+3 -3
View File
@@ -1,9 +1,9 @@
{"version":"2021.V.2.09",
{"version":"2021.V.2.10",
"build":"pass",
"test":"pass",
"grid_test":"pass",
"websites":"342",
"detections":"939",
"special":"2",
"special":"3",
"awaiting_verification":"22",
"auto_testing":"606e2f07-cf0d-4f05-98e0-bfd6ed204a1a"}
"auto_testing":"bb581703-6333-4d78-bb30-e20a9e846145"}
modules/externalapis.js
+153
View File
@@ -0,0 +1,153 @@
var helper = require("./helper.js")
var async = require("async");
async function get_words_info(all_words, words_info) {
var temp_added = []
for (let all_words_key of Object.keys(all_words)) {
for (let all_words_word of all_words[all_words_key]) {
if (!temp_added.includes(all_words_word)) {
temp_added.push(all_words_word);
var temp_words_info = {
"word": all_words_word,
"text": "",
"results": []
}
try {
var url1 = "https://api.duckduckgo.com/?q={0}&format=json&pretty=1&no_html=1&skip_disambig=1".replace("{0}", all_words_word);
var url2 = "https://api.duckduckgo.com/?q={0}&format=json&pretty=1".replace("{0}", all_words_word);
var response1 = await helper.get_url_wrapper_json(url1);
var response2 = await helper.get_url_wrapper_json(url2);
if (response2.data != '') {
if ("RelatedTopics" in response2.data) {
if (response2.data.RelatedTopics.length > 0) {
if (response2.data != '') {
if ("AbstractText" in response1.data && response1.data.AbstractText != "") {
temp_words_info.text = response1.data.AbstractText;
} else if ("Abstract" in response1.data && response1.data.AbstractText != "") {
temp_words_info.text = response1.data.Abstract;
} else {
temp_words_info.text = "unknown";
}
}
response2.data.RelatedTopics.forEach(function(item) {
if ("Name" in item) {
item.Topics.forEach(function(topic) {
temp_words_info.results.push({
"type": item.Name,
"text": topic.Text,
"url": topic.FirstURL
});
});
} else {
temp_words_info.results.push({
"type": "Related",
"text": item.Text,
"url": item.FirstURL
});
}
});
}
}
}
if (temp_words_info.results.length > 0) {
words_info.push(temp_words_info);
}
} catch (error) {
helper.verbose && console.log(error);
}
}
}
}
}
async function check_engines(req, info) {
try {
if (helper.google_api_key == "" || helper.google_api_cs == "") {
return
}
var url = "https://www.googleapis.com/customsearch/v1?key={0}&cx={1}&q={2}".replace("{0}", helper.google_api_key).replace("{1}", helper.google_api_cs).replace("{2}", req.body.string);
var response = await helper.get_url_wrapper_json(url);
if (response.data != '') {
try {
info.original = response.data.queries.request[0].searchTerms
} catch (e) {}
try {
info.corrected = response.data.spelling.correctedQuery
} catch (e) {}
try {
info.total = response.data.searchInformation.totalResults
} catch (e) {}
try {
response.data.items.forEach(function(item) {
info["items"].push({
"title": item.title,
"snippet": item.snippet
});
});
} catch (e) {}
try {
if (info.total == 0 && info.corrected != "") {
info.checking = info.original + " [Error]<br>Try this: " + info.corrected;
} else if (info.total > 0 && info.corrected != "") {
info.checking = info.original + " [Good]<br>Suggested word: " + info.corrected + "<br>Total lookups: " + info.total;
} else if (info.total > 0 && info.corrected == "") {
info.checking = info.original + " [Good]<br>Total lookups: " + info.total;
} else {
info.checking = "Using " + info.original + " with no lookups";
}
} catch (e) {}
}
} catch (error) {
helper.verbose && console.log(error);
}
}
async function custom_search_ouputs(req) {
var possible_parameters = ['user', 'profile', 'account']
const time = new Date();
const functions = [];
possible_parameters.forEach((key) => {
functions.push(custom_search_ouputs_website.bind(null, req.body.uuid, req.body.string, key));
});
const results = await async.parallelLimit(functions, 6);
helper.verbose && console.log(`Total time ${new Date() - time}`);
var merged = [].concat.apply([], results.filter(item => item !== undefined));
return merged
}
async function custom_search_ouputs_website(uuid, name, key) {
return new Promise(async (resolve, reject) => {
try {
var results = []
helper.log_to_file_queue(uuid, "[Custom Search Using] " + key)
var url = "https://www.googleapis.com/customsearch/v1?key={0}&cx={1}&q={2}:{3}".replace("{0}", helper.google_api_key).replace("{1}", helper.google_api_cs).replace("{2}", key).replace("{3}", name);
var response = await helper.get_url_wrapper_json(url);
if (response.data != '') {
if ('items' in response.data) {
response.data.items.forEach((key) => {
results.push({
site: helper.get_site_from_url(key.link),
link: key.link,
snippet: key.snippet
})
});
}
}
if (results.length > 0) {
resolve(results);
} else {
resolve(undefined)
}
} catch (err) {
resolve(undefined)
}
});
}
module.exports = {
check_engines,
get_words_info,
custom_search_ouputs
}
modules/fastscan.js
+98
View File
@@ -0,0 +1,98 @@
var helper = require("./helper.js")
var async = require("async");
var sanitizeHtml = require("sanitize-html");
const {
htmlToText
} = require('html-to-text');
var cheerio = require('cheerio');
async function find_username_normal(req) {
const time = new Date();
const functions = [];
await helper.parsed_sites.forEach(site => {
if ("status" in site) {
if (site.status == "bad") {
return Promise.resolve();
}
}
if (site.selected == "true" && site.detections.length > 0) {
functions.push(find_username_site.bind(null, req.body.uuid, req.body.string, req.body.option, site));
}
});
const results = await async.parallelLimit(functions, 100);
helper.verbose && console.log(`Total time ${new Date() - time}`);
return results.filter(item => item !== undefined)
}
async function find_username_site(uuid, username, options, site) {
return new Promise(async (resolve, reject) => {
try {
if (!options.includes('json')) {
helper.log_to_file_queue(uuid, "[Checking] " + helper.get_site_from_url(site.url))
}
var body = await helper.get_url_wrapper_text(site.url.replace("{username}", username));
var detections_count = 0;
var source = body;
var text_only = "unavailable";
var title = "unavailable";
var temp_profile = {
"found": 0,
"image": "",
"link": "",
"rate": "",
"title": "",
"text": "",
"type": ""
};
await Promise.all(site.detections.map(async detection => {
var temp_found = "false";
if (detection.type == "normal" && options.includes("FindUserProfilesFast") && source != "") {
detections_count += 1
if (source.toLowerCase().includes(detection.string.replace("{username}", username).toLowerCase())) {
temp_found = "true";
}
if (detection.return == temp_found) {
//console.log(detection.string, " > normal");
temp_profile.found += 1
}
}
}));
if (temp_profile.found > 0 && detections_count != 0) {
temp_profile.text = sanitizeHtml(htmlToText(body, {
wordwrap: false,
hideLinkHrefIfSameAsText: true,
ignoreHref: true,
ignoreImage: true
}));
if (temp_profile.text == "") {
temp_profile.text = "unavailable"
}
try {
var $ = cheerio.load(body);
title = sanitizeHtml($("title").text())
if (title.length == 0) {
title = "unavailable"
}
} catch (err) {
helper.verbose && console.log(err);
}
temp_profile.title = title;
temp_profile.rate = "%" + ((temp_profile["found"] / detections_count) * 100).toFixed(2);
temp_profile.link = site.url.replace("{username}", username);
temp_profile.type = site.type
resolve(temp_profile);
} else {
resolve(undefined)
}
} catch (err) {
resolve(undefined)
}
});
}
module.exports = {
find_username_normal
}
modules/helper.js
+112
View File
@@ -0,0 +1,112 @@
var verbose = false
var google_api_key = "";
var google_api_cs = "";
var grid_url = "";
var proxy = ""
var header_options = {
headers: {
'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:84.0) Gecko/20100101 Firefox/84.0',
}
};
var https = require("follow-redirects").https;
var fs = require("fs");
var url = require("url");
var parsed_sites = JSON.parse(fs.readFileSync("sites.json"));
var logs_queue = Promise.resolve();
function log_to_file_queue(uuid, msg) {
logs_queue = logs_queue.then(function() {
return new Promise(function(resolve) {
fs.appendFile("logs/" + uuid + "_log.txt", msg + "\n", function(err, data) {
console.log(msg)
resolve();
});
});
});
}
function get_site_from_url(_url) {
temp = url.parse(_url.replace("{username}", "nothinghere")).hostname
return temp.replace("nothinghere.", "")
}
async function get_url_wrapper_json(url, time = 5) {
try {
let http_promise = new Promise((resolve, reject) => {
var request = https.get(url, header_options, function(res) {
var body = ""
res.on("data", function(chunk) {
body += chunk;
});
res.on("end", function() {
resolve({
'data': JSON.parse(body.toString())
});
});
});
request.on('error', function(e) {
reject({
'data': ''
})
});
request.on('socket', function(socket) {
var timeout = (time != 0) ? time * 1000 : 5000;
socket.setTimeout(timeout, function() {
request.abort();
});
});
});
let response_body = await http_promise;
return response_body
} catch (err) {
verbose && console.log(err);
}
}
async function get_url_wrapper_text(url, time = 5) {
try {
let http_promise = new Promise((resolve, reject) => {
var request = https.get(url, header_options, function(res) {
var body = ""
res.on("data", function(chunk) {
body += chunk;
});
res.on("end", function() {
resolve(body);
});
});
request.on('error', function(e) {
reject({
'data': ''
})
});
request.on('socket', function(socket) {
var timeout = (time != 0) ? time * 1000 : 5000;
socket.setTimeout(timeout, function() {
request.abort();
});
});
});
let response_body = await http_promise;
return response_body
} catch (err) {
verbose && console.log(err);
}
}
module.exports = {
parsed_sites,
verbose,
google_api_key,
google_api_cs,
grid_url,
header_options,
proxy,
get_site_from_url,
log_to_file_queue,
get_url_wrapper_text,
get_url_wrapper_json
}
modules/nameanalysis.js
+83
View File
@@ -0,0 +1,83 @@
var helper = require("./helper.js")
var fs = require("fs");
var stringSimilarity = require("string-similarity");
var parsed_names_origins = JSON.parse(fs.readFileSync("names.json"));
async function find_origins(req) {
var found = []
for (key in parsed_names_origins) {
for (name in parsed_names_origins[key]['boy']) {
if (req.body.string.includes(parsed_names_origins[key]['boy'][name])) {
found.push({
"name": parsed_names_origins[key]['boy'][name],
"origin": key,
"gender": "boy",
"matched": parsed_names_origins[key]['boy'][name],
"similar": ""
})
} else {
var similarity = stringSimilarity.compareTwoStrings(req.body.string, parsed_names_origins[key]['boy'][name]);
if (similarity > 0.7) {
found.push({
"name": req.body.string,
"origin": key,
"gender": "boy",
"matched": "",
"similar": parsed_names_origins[key]['boy'][name]
})
}
}
}
for (name in parsed_names_origins[key]['girl']) {
if (req.body.string.includes(parsed_names_origins[key]['girl'][name])) {
found.push({
"name": parsed_names_origins[key]['girl'][name],
"origin": key,
"gender": "girl",
"matched": parsed_names_origins[key]['girl'][name],
"similar": ""
})
} else {
var similarity = stringSimilarity.compareTwoStrings(req.body.string, parsed_names_origins[key]['girl'][name]);
if (similarity > 0.7) {
found.push({
"name": req.body.string,
"origin": key,
"gender": "girl",
"matched": "",
"similar": parsed_names_origins[key]['girl'][name]
})
}
}
}
for (name in parsed_names_origins[key]['uni']) {
if (req.body.string.includes(parsed_names_origins[key]['uni'][name])) {
found.push({
"name": parsed_names_origins[key]['uni'][name],
"origin": key,
"gender": "uni",
"matched": parsed_names_origins[key]['uni'][name],
"similar": ""
})
} else {
var similarity = stringSimilarity.compareTwoStrings(req.body.string, parsed_names_origins[key]['uni'][name]);
if (similarity > 0.7) {
found.push({
"name": req.body.string,
"origin": key,
"gender": "uni",
"matched": "",
"similar": parsed_names_origins[key]['uni'][name]
})
}
}
}
}
return found
}
module.exports = {
find_origins
}
modules/slowscan.js
+182
View File
@@ -0,0 +1,182 @@
var helper = require("./helper.js")
var async = require("async");
var sanitizeHtml = require("sanitize-html");
var tesseract = require("node-tesseract-ocr");
var sanitizeHtml = require("sanitize-html");
var firefox = require("selenium-webdriver/firefox");
var {
Builder,
By,
Key
} = require("selenium-webdriver");
var tmp = require("tmp");
var fs = require("fs");
async function find_username_advanced(req) {
const time = new Date();
const functions = [];
helper.parsed_sites.forEach((site) => {
if ("status" in site) {
if (site.status == "bad") {
return Promise.resolve();
}
}
if (site.selected == "true" && site.detections.length > 0) {
functions.push(find_username_site.bind(null, req.body.uuid, req.body.string, req.body.option, site));
}
});
const results = await async.parallelLimit(functions, 8);
helper.verbose && console.log(`Total time ${new Date() - time}`);
return results.filter(item => item !== undefined)
}
async function find_username_site(uuid, username, options, site) {
return new Promise(async (resolve, reject) => {
helper.log_to_file_queue(uuid, "[Checking] " + helper.get_site_from_url(site.url))
let driver = undefined
if (helper.grid_url == "") {
driver = new Builder()
.forBrowser("firefox")
.setFirefoxOptions(new firefox.Options().headless().windowSize({
width: 640,
height: 480
}))
.build();
} else {
driver = new Builder()
.forBrowser("firefox")
.setFirefoxOptions(new firefox.Options().headless().windowSize({
width: 640,
height: 480
}))
.usingServer(helper.grid_url)
.build();
}
try {
var timeouts = {
implicit: 0,
pageLoad: 5000,
script: 5000
};
var timeout = (site.timeout != 0) ? site.timeout * 1000 : 5000;
var implicit = (site.implicit != 0) ? site.implicit * 1000 : 0;
timeouts = {
implicit: implicit,
pageLoad: timeout,
script: timeout
};
helper.verbose && console.log(timeouts)
var source = "";
var data = "";
var text_only = "unavailable";
var title = "unavailable";
var temp_profile = {
"found": 0,
"image": "",
"link": "",
"rate": "",
"title": "",
"text": "",
"type": ""
};
var link = site.url.replace("{username}", username);
await driver.manage().setTimeouts(timeouts);
await driver.get(link);;
source = await driver.getPageSource();
data = await driver.takeScreenshot();
title = await driver.getTitle();
text_only = await driver.findElement(By.tagName("body")).getText();
await driver.quit()
if (options.includes("ShowUserProfilesSlow")) {
temp_profile["image"] = "data:image/png;base64,{image}".replace("{image}", data);
}
if (site.selected == "true" && site.detections.length > 0 && options.includes("FindUserProfilesSlow")) {
await Promise.all(site.detections.map(async detection => {
try {
if ("status" in detection) {
if (detection.status == "bad") {
return;
}
}
var temp_found = "false"
if (detection.type == "ocr" && data != "") {
tmpobj = tmp.fileSync();
fs.writeFileSync(tmpobj.name, Buffer.from(data, "base64"));
await tesseract.recognize(tmpobj.name, {
lang: "eng",
oem: 1,
psm: 3,
})
.then(text => {
text = text.replace(/[^A-Za-z0-9]/gi, "");
detection.string = detection.string.replace(/[^A-Za-z0-9]/gi, "");
if (text != "") {
if (text.toLowerCase().includes(detection.string.toLowerCase())) {
temp_found = "true";
}
if (detection.return == temp_found) {
//console.log(text);
//console.log(detection.string," > Found ocr");
temp_profile.found += 1;
}
}
})
.catch(error => {
helper.verbose && console.log(error.message);
})
tmpobj.removeCallback();
} else if (detection.type == "normal" && source != "") {
if (source.toLowerCase().includes(detection.string.replace("{username}", username).toLowerCase())) {
temp_found = "true";
}
if (detection.return == temp_found) {
//console.log(detection.string," > normal");
temp_profile.found += 1
}
} else if (detection.type == "advanced" && text_only != "") {
if (text_only.toLowerCase().includes(detection.string.replace("{username}", username).toLowerCase())) {
temp_found = "true";
}
if (detection.return == temp_found) {
//console.log(detection.string," > normal");
temp_profile.found += 1
}
}
} catch (err) {
}
}));
}
if (temp_profile.found > 0 || temp_profile.image != "") {
temp_profile.text = sanitizeHtml(text_only);
temp_profile.title = sanitizeHtml(title);
temp_profile.rate = "%" + ((temp_profile.found / site.detections.length) * 100).toFixed(2);
temp_profile.link = site.url.replace("{username}", username);
temp_profile.type = site.type
resolve(temp_profile);
} else {
resolve(undefined)
}
} catch (err) {
if (driver !== undefined) {
try {
await driver.quit()
} catch (err) {
helper.verbose && console.log("Driver Session Issue")
}
}
resolve(undefined)
}
});
}
module.exports = {
find_username_advanced
}
modules/special.js
+241
View File
@@ -0,0 +1,241 @@
var helper = require("./helper.js")
var async = require("async");
var sanitizeHtml = require("sanitize-html");
var tesseract = require("node-tesseract-ocr");
var sanitizeHtml = require("sanitize-html");
var firefox = require("selenium-webdriver/firefox");
var {
Builder,
By,
Key
} = require("selenium-webdriver");
async function find_username_special(req) {
const time = new Date();
const functions = [];
helper.parsed_sites.forEach((site) => {
if ("status" in site) {
if (site.status == "bad") {
return Promise.resolve();
}
}
if (site.selected == "true") {
site.detections.forEach((detection) => {
if (detection.type == 'special') {
if (detection.function == 'special_facebook_1') {
functions.push(find_username_site_special_facebook_1.bind(null, req.body.uuid, req.body.string, site));
} else if (detection.function == 'special_gmail_1') {
functions.push(find_username_site_special_gmail_1.bind(null, req.body.uuid, req.body.string, site));
} else if (detection.function == 'special_google_1') {
functions.push(find_username_site_special_google_1.bind(null, req.body.uuid, req.body.string, site));
}
}
});
}
});
const results = await async.parallelLimit(functions, 5);
helper.verbose && console.log(`Total time ${new Date() - time}`);
return results.filter(item => item !== undefined)
}
async function find_username_site_special_facebook_1(uuid, username, site) {
return new Promise(async (resolve, reject) => {
helper.log_to_file_queue(uuid, "[Checking] " + helper.get_site_from_url(site.url))
let driver = new Builder()
.forBrowser("firefox")
.setFirefoxOptions(new firefox.Options().headless().windowSize({
width: 640,
height: 480
}))
.build();
try {
var timeouts = {
implicit: 0,
pageLoad: 10000,
script: 10000
};
var source = "";
var data = "";
var text_only = "unavailable";
var title = "unavailable";
var temp_profile = {
"found": 0,
"image": "",
"link": "",
"rate": "",
"title": "",
"text": "",
"type": ""
};
var link = "https://mbasic.facebook.com/login/identify/?ctx=recoveqr";
await driver.manage().setTimeouts(timeouts);
await driver.get(link);;
await driver.findElement(By.id('identify_search_text_input')).sendKeys(username);
await driver.findElement(By.id('did_submit')).click();
source = await driver.getPageSource();
text_only = await driver.findElement(By.tagName("body")).getText();
await driver.quit()
if (source.includes("Try Entering Your Password")) {
temp_found = "true";
temp_profile.found += 1
}
if (temp_profile.found > 0) {
temp_profile.text = "unavailable";
temp_profile.title = "unavailable";
temp_profile.rate = "%" + ((temp_profile.found / 1) * 100).toFixed(2);
temp_profile.link = site.url.replace("{username}", username);
temp_profile.type = site.type
resolve(temp_profile);
} else {
resolve(undefined)
}
} catch (err) {
if (driver !== undefined) {
try {
await driver.quit()
} catch (err) {
helper.verbose && console.log("Driver Session Issue")
}
}
resolve(undefined)
}
});
}
async function find_username_site_special_gmail_1(uuid, username, site) {
return new Promise(async (resolve, reject) => {
helper.log_to_file_queue(uuid, "[Checking] " + helper.get_site_from_url(site.url))
let driver = new Builder()
.forBrowser("firefox")
.setFirefoxOptions(new firefox.Options().headless().windowSize({
width: 640,
height: 480
}))
.build();
try {
var timeouts = {
implicit: 0,
pageLoad: 10000,
script: 10000
};
var source = "";
var data = "";
var text_only = "unavailable";
var title = "unavailable";
var temp_profile = {
"found": 0,
"image": "",
"link": "",
"rate": "",
"title": "",
"text": "",
"type": ""
};
var link = "https://accounts.google.com/signup/v2/webcreateaccount?service=mail&continue=https%3A%2F%2Fmail.google.com%2Fmail%2F%3Fpc%3Dtopnav-about-n-en&flowName=GlifWebSignIn&flowEntry=SignUp";
await driver.manage().setTimeouts(timeouts);
await driver.get(link);;
await driver.findElement(By.id('username')).sendKeys(username);
await driver.findElement(By.id('selectioni1')).click();
source = await driver.getPageSource();
text_only = await driver.findElement(By.tagName("body")).getText();
await driver.quit()
if (text_only.includes("That username is taken") && !text_only.includes('your username must be between') && !text_only.includes('You can use letters')) {
temp_found = "true";
temp_profile.found += 1
}
if (temp_profile.found > 0) {
temp_profile.text = username + "@gmail.com";
temp_profile.title = "unavailable";
temp_profile.rate = "%" + ((temp_profile.found / 1) * 100).toFixed(2);
temp_profile.link = "https://google.com"
temp_profile.type = site.type
resolve(temp_profile);
} else {
resolve(undefined)
}
} catch (err) {
if (driver !== undefined) {
try {
await driver.quit()
} catch (err) {
helper.verbose && console.log("Driver Session Issue")
}
}
resolve(undefined)
}
});
}
async function find_username_site_special_google_1(uuid, username, site) {
return new Promise(async (resolve, reject) => {
helper.log_to_file_queue(uuid, "[Checking] " + helper.get_site_from_url(site.url))
let driver = new Builder()
.forBrowser("firefox")
.setFirefoxOptions(new firefox.Options().headless().windowSize({
width: 640,
height: 480
}))
.build();
try {
var timeouts = {
implicit: 0,
pageLoad: 10000,
script: 10000
};
var source = "";
var data = "";
var text_only = "unavailable";
var title = "unavailable";
var temp_profile = {
"found": 0,
"image": "",
"link": "",
"rate": "",
"title": "",
"text": "",
"type": ""
};
var link = "https://accounts.google.com/signin/v2/identifier?continue=https%3A%2F%2Fmail.google.com%2Fmail%2F&service=mail&sacu=1&rip=1&flowName=GlifWebSignIn&flowEntry=ServiceLogin";
await driver.manage().setTimeouts(timeouts);
await driver.get(link);;
await driver.findElement(By.id('identifierId')).sendKeys(username);
await driver.findElement(By.xpath("//button[contains(.,'Next')]")).click();
source = await driver.getPageSource();
text_only = await driver.findElement(By.tagName("body")).getText();
await driver.quit()
if (text_only.includes("Couldn't sign you in") && !text_only.includes("Couldn't find your")) {
temp_found = "true";
temp_profile.found += 1
}
if (temp_profile.found > 0) {
temp_profile.text = username + "@gmail.com";
temp_profile.title = "unavailable";
temp_profile.rate = "%" + ((temp_profile.found / 1) * 100).toFixed(2);
temp_profile.link = "https://google.com"
temp_profile.type = site.type
resolve(temp_profile);
} else {
resolve(undefined)
}
} catch (err) {
if (driver !== undefined) {
try {
await driver.quit()
} catch (err) {
helper.verbose && console.log("Driver Session Issue")
}
}
resolve(undefined)
}
});
}
module.exports = {
find_username_special
}
modules/stringanalysis.js
+229
View File
@@ -0,0 +1,229 @@
var helper = require("./helper.js")
var {
findWord
} = require("most-common-words-by-language");
var fs = require("fs");
var parsed_json = JSON.parse(fs.readFileSync("dict.json"));
var WordsNinjaPack = require("wordsninja");
var WordsNinja = new WordsNinjaPack();
function most_common1(all_words, temp_words) {
var temp_list = []
Object.keys(all_words).forEach(function(key) {
all_words[key].forEach(function(item) {
if (!temp_list.includes(item) && item.length > 1) {
temp_list.push(item);
var temp = findWord(item);
if (Object.keys(temp).length != 0) {
var languages = Object.keys(temp).map(function(key) {
return [key, temp[key]];
});
languages.sort(function(first, second) {
return second[1] - first[1]
}).reverse();
temp_words.push({
"word": item,
"languages": languages.map(e => e.join(":")).join(" ")
});
}
}
});
});
}
async function most_common(all_words, temp_words) {
var temp_list = []
Object.keys(all_words).forEach(function(key) {
all_words[key].forEach(function(item) {
if (!temp_list.includes(item) && item.length > 1) {
temp_list.push(item);
var temp = findWord(item);
if (Object.keys(temp).length != 0) {
var languages = Object.keys(temp).map(function(key) {
return [key, temp[key]];
});
languages.sort(function(first, second) {
return second[1] - first[1]
}).reverse();
temp_words.push({
"word": item,
"languages": languages.map(e => e[0]).join(", ")
});
}
}
});
});
}
function find_other(all_words) {
var words = WordsNinja.splitSentence(req.body.string);
words.forEach(function(word) {
var value = false
Object.keys(all_words).forEach(function(key) {
if (all_words[key].includes(word)) {
value = true
}
});
if (!value && !all_words.maybe.includes(word)) {
all_words.maybe.push(word);
}
});
}
function remove_word(str, sub_string) {
part1 = str.substring(0, str.indexOf(sub_string));
part2 = str.substring(str.indexOf(sub_string) + sub_string.length, str.length);
temp = (part1 + part2).replace(/[ \[\]:"\\|,.<>\/?~`!@#$%^&*()_+\-={};"]/gi, "");
return temp;
}
async function analyze_string(req, all_words) {
helper.log_to_file_queue(req.body.uuid, "[Starting] String analysis")
temp_rr_names = []
string_to_check = req.body.string
parsed_json.prefix.forEach(function(item, index) {
if (string_to_check.indexOf(item) == 0 && !all_words.prefix.includes(item)) {
all_words.prefix.push(item);
temp = remove_word(string_to_check, item);
if (temp !== null && temp !== "" && !all_words.unknown.includes(temp) && !all_words.maybe.includes(temp) && temp.length > 1) {
all_words.unknown.push(temp);
}
}
});
parsed_json.m_names.forEach(function(item, index) {
if (string_to_check.indexOf(item) >= 0 && !all_words.name.includes(item)) {
all_words.name.push(item);
temp = remove_word(string_to_check, item);
if (temp !== null && temp !== "" && !all_words.unknown.includes(temp) && !all_words.maybe.includes(temp) && temp.length > 1) {
all_words.unknown.push(temp);
}
}
});
parsed_json.f_names.forEach(function(item, index) {
if (string_to_check.indexOf(item) >= 0 && !all_words.name.includes(item)) {
all_words.name.push(item);
temp = remove_word(string_to_check, item);
if (temp !== null && temp !== "" && !all_words.unknown.includes(temp) && !all_words.maybe.includes(temp) && temp.length > 1) {
all_words.unknown.push(temp);
}
}
});
all_words.prefix.forEach(function(h_item, index) {
all_words.unknown.forEach(function(r_item, index) {
if (r_item.indexOf(h_item) == 0) {
temp = remove_word(r_item, h_item);
if (temp !== null && temp !== "" && !temp_rr_names.includes(temp) && !all_words.maybe.includes(temp) && temp.length > 1) {
temp_rr_names.push(temp);
}
}
});
});
var temp_r_concat = all_words.unknown.concat(temp_rr_names.filter((item) => all_words.unknown.indexOf(item) < 0));
all_words.unknown = temp_r_concat
temp_rr_names = []
all_words.number.forEach(function(n_item, index) {
all_words.unknown.forEach(function(r_item, index) {
if (r_item.indexOf(n_item) >= 0) {
temp = remove_word(r_item, n_item);
if (temp !== null && temp !== "" && !temp_rr_names.includes(temp) && !all_words.maybe.includes(temp) && temp.length > 1) {
temp_rr_names.push(temp);
}
}
});
});
var temp_r_concat = all_words.unknown.concat(temp_rr_names.filter((item) => all_words.unknown.indexOf(item) < 0));
all_words.unknown = temp_r_concat
helper.log_to_file_queue(req.body.uuid, "[Done] String analysis")
}
async function split_upper_case(req, all_words) {
try {
req.body.string.match(/[A-Z][a-z]+/g).forEach((item) => {
if (item.length > 1 && !all_words.unknown.includes(item) && !all_words.maybe.includes(item)) {
all_words.unknown.push(item.toLowerCase());
}
});
} catch (err) {}
}
async function split_alphabet_case(req, all_words) {
try {
req.body.string.match(/[A-Za-z]+/g).forEach((item) => {
if (item.length > 1 && !all_words.unknown.includes(item) && !all_words.maybe.includes(item)) {
all_words.unknown.push(item.toLowerCase());
}
});
} catch (err) {}
}
async function find_symbols(req, all_words) {
try {
req.body.string.match(/[ \[\]:"\\|,.<>\/?~`!@#$%^&*()_+\-={};']/gi).forEach((item) => {
if (item !== " " && !all_words.symbol.includes(item)) {
all_words.symbol.push(item);
}
});
} catch (err) {}
}
async function find_numbers(req, all_words) {
try {
req.body.string.match(/(\d+)/g).forEach((item) => {
if (!all_words.number.includes(item)) {
all_words.number.push(item);
}
});
} catch (err) {}
}
async function convert_numbers(req) {
try {
numbers_to_letters = {
"4": "a",
"8": "b",
"3": "e",
"1": "l",
"0": "o",
"5": "s",
"7": "t",
"2": "z"
}
temp_value = ""
for (i = 0; i < req.body.string.length; i++) {
_temp = numbers_to_letters[req.body.string.charAt(i)]
if (_temp != undefined) {
temp_value += numbers_to_letters[req.body.string.charAt(i)];
} else {
temp_value += req.body.string.charAt(i);
}
}
req.body.string = temp_value
} catch (err) {}
}
async function get_maybe_words(req, all_words) {
await WordsNinja.loadDictionary();
all_words.maybe = await WordsNinja.splitSentence(req.body.string).filter(function(elem, index, self) {
return index === self.indexOf(elem);
}).filter(word => word.length > 1);
}
module.exports = {
get_maybe_words,
find_symbols,
find_numbers,
convert_numbers,
split_upper_case,
split_alphabet_case,
most_common,
find_other,
analyze_string
}
package.json
+4 -4
View File
@@ -1,14 +1,15 @@
{
"name": "social-analyzer",
"version": "2.0.2",
"version": "2.0.10",
"description": "Advanced Social Media Profiles Finder and String Analysis Tool",
"main": "server.js",
"main": "app.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1",
"start": "node app.js"
},
"dependencies": {
"async": "3.2.0",
"cheerio": "1.0.0-rc.5",
"express": "^4.16.3",
"express-async-errors": "3.1.1",
"follow-redirects": "1.13.0",
@@ -28,8 +29,7 @@
"tmp": "0.2.1",
"wink-tokenizer": "1.1.0",
"wordsninja": "1.0.0",
"yargs": "16.2.0",
"cheerio": "1.0.0-rc.5"
"yargs": "16.2.0"
},
"author": "QeeqBox",
"license": "AGPL-3.0",
public/app.html
+2 -2
View File
@@ -1,6 +1,6 @@
<!--
// -------------------------------------------------------------
// author Giga
// author Giga
// project qeeqbox/social-analyzer
// email gigaqeeq@gmail.com
// description app.py (CLI)
@@ -909,7 +909,7 @@
await $.ajax({
type: "POST",
url: "/url",
url: "/analyze_string",
data: {
'string': $('#string-to-analyze').val(),
'option': list_of_options.join(","),
sites.json
+18 -2
View File
@@ -2160,6 +2160,22 @@
"implicit": 0,
"type": "Social networking service"
},
{
"url": "https://gmail.com/{username}",
"detections": [
{
"return": "true",
"string": "",
"function": "special_gmail_1",
"type": "special"
}
],
"selected": "false",
"top": "top10",
"timeout": 0,
"implicit": 0,
"type": "Social networking service"
},
{
"url": "https://google.com/{username}",
"detections": [
@@ -2170,7 +2186,7 @@
"type": "special"
}
],
"selected": "true",
"selected": "false",
"top": "top10",
"timeout": 0,
"implicit": 0,
@@ -7226,7 +7242,7 @@
"type": "normal"
}
],
"selected": "true",
"selected": "false",
"top": "top10",
"timeout": 0,
"implicit": 0,