diff --git a/README.md b/README.md
index 841d977..2d2e01f 100644
--- a/README.md
+++ b/README.md
@@ -5,18 +5,18 @@
API and Web App for analyzing & finding a person profile across +300 social media websites. It includes different string analysis and detection modules, you can choose which combination of modules to use during the investigation.
-The detection modules utilize a rating mechanism based on different detection techniques, which produces a rate value that starts from 0 to 100 (No-Maybe-Yes)
+The detection modules utilize a rating mechanism based on different detection techniques, which produces a rate value that starts from 0 to 100 (No-Maybe-Yes). This model produces less *False Positive* unlike other projects
This project could help in investigating profiles related to suspicious or malicious activities such as [cyberbullying](https://en.wikipedia.org/wiki/Wikipedia:Cyberbullying), [cybergrooming](https://de.wikipedia.org/wiki/Cyber-Grooming) & [cyberstalking](https://en.wikipedia.org/wiki/Cyberstalking), [spreading misinformation](https://en.wikipedia.org/wiki/Misinformation). And, *"It's currently used by some law enforcement agencies in countries where resources are limited"*
**Pull Requests are welcomed!**
## Updates
-- (Testing) A new refactored version (Coming soon!!)
-- Added special detection for google 👏
-- Added more customization to the CLI python version (It's a lot easier to integrate using the JSON output switch) 👏
-- Added Custom Search Query 👏
-- Added similarity checking to FindOrigins 👏
+- Added a new refactored version 👏👏👏
+- Added special detection for google
+- Added more customization to the CLI python version (It's a lot easier to integrate using the JSON output switch)
+- Added Custom Search Query
+- Added similarity checking to FindOrigins
## Security Testing
@@ -52,11 +52,11 @@ Profile images **will not** be blurred. If you want them to be blurred, turn tha
- Custom user-agent, proxy, timeout & implicit wait
- Python CLI & NodeJS CLI (Limited to FindUserProfilesFast option)
- Grid Option for Faster Checking (Limited to docker-compose) 👏
-- Dump Logs to folder or terminal
+- Dump Logs to folder or terminal (prettified)
- [Wiki](https://github.com/qeeqbox/social-analyzer/wiki)
## Special Detections
-- Facebook
+- Facebook, @gmail and google
## Install and run as web app (NodeJS + NPM + Firefox)
```bash
@@ -81,9 +81,9 @@ cd social-analyzer
rm -rf package-lock.json node_modules
npm install lodash
npm install
-# If you want to list all websites use node app.js -c -l
+# If you want to list all websites use node app.js --cli --list
# Remember the following runs as FindUserProfilesFast
-node app.js -c -m "fast" -u "username" -w "youtube pinterest tumblr"
+node app.j --cli --mode "fast" --username "johndoe" --websites "youtube pinterest tumblr" --output "pretty"
```
## Install and run as CLI (Python3 + NPM + Firefox)
@@ -94,7 +94,7 @@ sudo apt-get install -y firefox-esr tesseract-ocr git
git clone https://github.com/qeeqbox/social-analyzer.git
cd social-analyzer
pip3 install lxml BeautifulSoup4 tld pygments
-# If you want to list all websites use python3 app.py -c -l
+# If you want to list all websites use python3 app.py --cli --list
# Remember the following runs as FindUserProfilesFast
python3 app.py --cli --mode "fast" --username "johndoe" --websites "youtube pinterest tumblr" --output "pretty"
```
diff --git a/app.js b/app.js
index 051ecd7..ee841ea 100644
--- a/app.js
+++ b/app.js
@@ -8,145 +8,74 @@
// contributors list qeeqbox/social-analyzer/graphs/contributors
// -------------------------------------------------------------
-var google_api_key = "";
-var google_api_cs = "";
-var grid_url = "";
-var verbose = false
-var proxy = ""
+var argv = require('yargs')
+ .usage('Usage: $0 --cli --mode "fast" --username "johndoe" --websites "youtube tiktok"\nUsage: $0 --cli --mode "fast" --username "johndoe"')
+ .describe('cli', 'a user or stirng')
+ .default("cli", false)
+ .boolean('cli')
+ .describe('username', 'E.g. johndoe, john_doe or johndoe9999')
+ .default("username", "")
+ .describe('websites', 'Website or websites separated by space E.g. youtube, tiktok or tumblr')
+ .default("websites", "all")
+ .describe('mode', 'Analysis mode E.g.fast -> FindUserProfilesFast, slow -> FindUserProfilesSlow or special -> FindUserProfilesSpecial')
+ .default("mode", "")
+ .describe('output', 'arg_parser_optional.add_argument("--output", help="Show the output in the following format: json -> json output for integration or pretty -> prettify the output')
+ .default("output", "")
+ .describe('options', 'Show the following when a profile is found: link, rate, title or text')
+ .default("options", "")
+ .describe('list', 'List all available websites')
+ .default("list", false)
+ .boolean('list')
+ .describe('grid', 'grid option, not for CLI')
+ .default("grid", "")
+ .help('help')
+ .argv;
-var header_options = {
- headers: {
- 'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:84.0) Gecko/20100101 Firefox/84.0',
- }
-};
-
-console.log('[!] Detections are updated every often, make sure to get the most updated ones');
+if (argv.output != "json") {
+ console.log('[!] Detections are updated every often, make sure to get the most updated ones');
+}
var semver = require('semver');
if (semver.satisfies(process.version, '>13 || <13')) {
- console.log('[Good] NodeJS Version Check');
+ if (argv.output != "json") {
+ console.log('[Good] NodeJS Version Check');
+ }
} else {
- console.log('[Error] NodeJS Version Check');
+ if (argv.output != "json") {
+ console.log('[Error] NodeJS Version Check');
+ }
process.exit(1);
}
-var argv = require('yargs')
- .alias('c', 'cli')
- .alias('u', 'user')
- .alias('w', 'website')
- .alias('o', 'output')
- .alias('l', 'list')
- .alias('m', 'mode')
- .usage('Usage: $0 -c -m "mode" -u "user" -w "website[s]" -o "output"')
- .example('$0 -c -m "fast" -u "joe" -w "facebook"')
- .example('$0 -c -m "fast" -u "natalie" -w "facebook wordpress"')
- .describe('u', 'a user or stirng')
- .describe('w', 'a website or websites sparated with space')
- .describe('o', 'option output file')
- .describe('l', 'list all available websites')
- .describe('m', 'fast -> FindUserProfilesFast\nslow -> FindUserProfilesSlow\nspecial -> FindUserProfilesSpecial')
- .help('h')
- .alias('h', 'help')
- .argv;
-
-var tmp = require("tmp");
var express = require("express");
var fs = require("fs");
var tokenizer = require("wink-tokenizer");
-var WordsNinjaPack = require("wordsninja");
var generatorics = require("generatorics");
-var {
- findWord
-} = require("most-common-words-by-language");
-var tesseract = require("node-tesseract-ocr");
-var url = require("url");
-var sanitizeHtml = require("sanitize-html");
-var firefox = require("selenium-webdriver/firefox");
-var {
- Builder,
- By,
- Key
-} = require("selenium-webdriver");
-var util = require('util');
-var https = require("follow-redirects").https;
-var async = require("async");
var HttpsProxyAgent = require('https-proxy-agent');
var PrettyError = require('pretty-error');
-var stringSimilarity = require("string-similarity");
var pe = new PrettyError();
require('express-async-errors');
-//var jsdom = require('jsdom');
-//var dom = new jsdom.JSDOM();
-//var window = dom.window;
-//var document = window.document;
-//var $ = require('jquery')(window);
-const {
- htmlToText
-} = require('html-to-text');
-var cheerio = require('cheerio');
var _tokenizer = tokenizer();
-var parsed_json = JSON.parse(fs.readFileSync("dict.json"));
-var parsed_sites = JSON.parse(fs.readFileSync("sites.json"));
-var parsed_names_origins = JSON.parse(fs.readFileSync("names.json"));
-var app = express();
-
-var WordsNinja = new WordsNinjaPack();
-app.use(express.urlencoded({
- extended: true
-}));
-app.use(express.json());
-app.use(express.static("public"));
+var helper = require("./modules/helper.js")
+var fastscan = require("./modules/fastscan.js")
+var slowscan = require("./modules/slowscan.js")
+var special = require("./modules/special.js")
+var externalapis = require("./modules/externalapis.js")
+var stringanalysis = require("./modules/stringanalysis.js")
+var nameanalysis = require("./modules/nameanalysis.js")
if (!fs.existsSync('logs')) {
fs.mkdirSync('logs');
}
-var logs_queue = Promise.resolve();
+var app = express();
-function log_to_file_queue(uuid, msg) {
- logs_queue = logs_queue.then(function() {
- return new Promise(function(resolve) {
- fs.appendFile("logs/" + uuid + "_log.txt", msg + "\n", function(err, data) {
- console.log(msg)
- resolve();
- });
- });
- });
-}
-
-async function get_url_wrapper_json(url, time = 5) {
- try {
- let http_promise = new Promise((resolve, reject) => {
- var request = https.get(url, header_options, function(res) {
- var body = ""
- res.on("data", function(chunk) {
- body += chunk;
- });
- res.on("end", function() {
- resolve({
- 'data': JSON.parse(body.toString())
- });
- });
- });
- request.on('error', function(e) {
- reject({
- 'data': ''
- })
- });
- request.on('socket', function(socket) {
- var timeout = (time != 0) ? time * 1000 : 5000;
- socket.setTimeout(timeout, function() {
- request.abort();
- });
- });
- });
- let response_body = await http_promise;
- return response_body
- } catch (err) {
- verbose && console.log(err);
- }
-}
+app.use(express.urlencoded({
+ extended: true
+}));
+app.use(express.json());
+app.use(express.static("public"));
app.post("/get_logs", async function(req, res, next) {
var last_line = "nothinghere"
@@ -160,610 +89,9 @@ app.post("/get_logs", async function(req, res, next) {
}
})
-function get_site_from_url(_url) {
- temp = url.parse(_url.replace("{username}", "nothinghere")).hostname
- return temp.replace("nothinghere.", "")
-}
-
-async function find_username_special(req) {
- const time = new Date();
- const functions = [];
- parsed_sites.forEach((site) => {
- if ("status" in site) {
- if (site.status == "bad") {
- return Promise.resolve();
- }
- }
- if (site.selected == "true") {
- site.detections.forEach((detection) => {
- if (detection.type == 'special') {
- if (detection.function == 'special_facebook_1') {
- functions.push(find_username_site_special_facebook_1.bind(null, req.body.uuid, req.body.string, site));
- } else if (detection.function == 'special_google_1') {
- functions.push(find_username_site_special_google_1.bind(null, req.body.uuid, req.body.string, site));
- }
- }
- });
- }
- });
- const results = await async.parallelLimit(functions, 5);
- verbose && console.log(`Total time ${new Date() - time}`);
- return results.filter(item => item !== undefined)
-}
-
-async function find_username_site_special_facebook_1(uuid, username, site) {
- return new Promise(async (resolve, reject) => {
- log_to_file_queue(uuid, "[Checking] " + get_site_from_url(site.url))
- let driver = new Builder()
- .forBrowser("firefox")
- .setFirefoxOptions(new firefox.Options().headless().windowSize({
- width: 640,
- height: 480
- }))
- .build();
-
- try {
- var timeouts = {
- implicit: 0,
- pageLoad: 10000,
- script: 10000
- };
-
- var source = "";
- var data = "";
- var text_only = "unavailable";
- var title = "unavailable";
- var temp_profile = {
- "found": 0,
- "image": "",
- "link": "",
- "rate": "",
- "title": "",
- "text": "",
- "type": ""
- };
- var link = "https://mbasic.facebook.com/login/identify/?ctx=recoveqr";
- await driver.manage().setTimeouts(timeouts);
- await driver.get(link);;
- await driver.findElement(By.id('identify_search_text_input')).sendKeys(username);
- await driver.findElement(By.id('did_submit')).click();
- source = await driver.getPageSource();
- text_only = await driver.findElement(By.tagName("body")).getText();
- await driver.quit()
- if (source.includes("Try Entering Your Password")) {
- temp_found = "true";
- temp_profile.found += 1
- }
- if (temp_profile.found > 0) {
- temp_profile.text = "unavailable";
- temp_profile.title = "unavailable";
- temp_profile.rate = "%" + ((temp_profile.found / 1) * 100).toFixed(2);
- temp_profile.link = site.url.replace("{username}", username);
- temp_profile.type = site.type
- resolve(temp_profile);
- } else {
- resolve(undefined)
- }
- } catch (err) {
- if (driver !== undefined) {
- try {
- await driver.quit()
- } catch (err) {
- verbose && console.log("Driver Session Issue")
- }
- }
- resolve(undefined)
- }
- });
-}
-
-async function find_username_site_special_google_1(uuid, username, site) {
- return new Promise(async (resolve, reject) => {
- log_to_file_queue(uuid, "[Checking] " + get_site_from_url(site.url))
- let driver = new Builder()
- .forBrowser("firefox")
- .setFirefoxOptions(new firefox.Options().headless().windowSize({
- width: 640,
- height: 480
- }))
- .build();
-
- try {
- var timeouts = {
- implicit: 0,
- pageLoad: 10000,
- script: 10000
- };
-
- var source = "";
- var data = "";
- var text_only = "unavailable";
- var title = "unavailable";
- var temp_profile = {
- "found": 0,
- "image": "",
- "link": "",
- "rate": "",
- "title": "",
- "text": "",
- "type": ""
- };
- var link = "https://accounts.google.com/signup/v2/webcreateaccount?service=mail&continue=https%3A%2F%2Fmail.google.com%2Fmail%2F%3Fpc%3Dtopnav-about-n-en&flowName=GlifWebSignIn&flowEntry=SignUp";
- await driver.manage().setTimeouts(timeouts);
- await driver.get(link);;
- await driver.findElement(By.id('username')).sendKeys(username);
- await driver.findElement(By.id('selectioni1')).click();
- source = await driver.getPageSource();
- text_only = await driver.findElement(By.tagName("body")).getText();
- await driver.quit()
- if (text_only.includes("That username is taken") && !text_only.includes('your username must be between') && !text_only.includes('You can use letters')) {
- temp_found = "true";
- temp_profile.found += 1
- }
- if (temp_profile.found > 0) {
- temp_profile.text = username + "@gmail.com";
- temp_profile.title = "unavailable";
- temp_profile.rate = "%" + ((temp_profile.found / 1) * 100).toFixed(2);
- temp_profile.link = "https://google.com"
- temp_profile.type = site.type
- resolve(temp_profile);
- } else {
- resolve(undefined)
- }
- } catch (err) {
- if (driver !== undefined) {
- try {
- await driver.quit()
- } catch (err) {
- verbose && console.log("Driver Session Issue")
- }
- }
- resolve(undefined)
- }
- });
-}
-
-async function find_username_advanced(req) {
- const time = new Date();
- const functions = [];
- parsed_sites.forEach((site) => {
- if ("status" in site) {
- if (site.status == "bad") {
- return Promise.resolve();
- }
- }
- if (site.selected == "true" && site.detections.length > 0) {
- functions.push(find_username_site_new.bind(null, req.body.uuid, req.body.string, req.body.option, site));
- }
- });
- const results = await async.parallelLimit(functions, 8);
- verbose && console.log(`Total time ${new Date() - time}`);
- return results.filter(item => item !== undefined)
-}
-
-async function find_username_site_new(uuid, username, options, site) {
- return new Promise(async (resolve, reject) => {
- log_to_file_queue(uuid, "[Checking] " + get_site_from_url(site.url))
- let driver = undefined
- if (grid_url == "") {
- driver = new Builder()
- .forBrowser("firefox")
- .setFirefoxOptions(new firefox.Options().headless().windowSize({
- width: 640,
- height: 480
- }))
- .build();
- } else {
- driver = new Builder()
- .forBrowser("firefox")
- .setFirefoxOptions(new firefox.Options().headless().windowSize({
- width: 640,
- height: 480
- }))
- .usingServer(grid_url)
- .build();
- }
-
-
- try {
-
- var timeouts = {
- implicit: 0,
- pageLoad: 5000,
- script: 5000
- };
-
- var timeout = (site.timeout != 0) ? site.timeout * 1000 : 5000;
- var implicit = (site.implicit != 0) ? site.implicit * 1000 : 0;
-
- timeouts = {
- implicit: implicit,
- pageLoad: timeout,
- script: timeout
- };
-
- verbose && console.log(timeouts)
-
- var source = "";
- var data = "";
- var text_only = "unavailable";
- var title = "unavailable";
- var temp_profile = {
- "found": 0,
- "image": "",
- "link": "",
- "rate": "",
- "title": "",
- "text": "",
- "type": ""
- };
- var link = site.url.replace("{username}", username);
- await driver.manage().setTimeouts(timeouts);
- await driver.get(link);;
- source = await driver.getPageSource();
- data = await driver.takeScreenshot();
- title = await driver.getTitle();
- text_only = await driver.findElement(By.tagName("body")).getText();
- await driver.quit()
- if (options.includes("ShowUserProfilesSlow")) {
- temp_profile["image"] = "data:image/png;base64,{image}".replace("{image}", data);
- }
- if (site.selected == "true" && site.detections.length > 0 && options.includes("FindUserProfilesSlow")) {
- await Promise.all(site.detections.map(async detection => {
- try {
- if ("status" in detection) {
- if (detection.status == "bad") {
- return;
- }
- }
- var temp_found = "false"
- if (detection.type == "ocr" && data != "") {
- tmpobj = tmp.fileSync();
- fs.writeFileSync(tmpobj.name, Buffer.from(data, "base64"));
- await tesseract.recognize(tmpobj.name, {
- lang: "eng",
- oem: 1,
- psm: 3,
- })
- .then(text => {
- text = text.replace(/[^A-Za-z0-9]/gi, "");
- detection.string = detection.string.replace(/[^A-Za-z0-9]/gi, "");
- if (text != "") {
- if (text.toLowerCase().includes(detection.string.toLowerCase())) {
- temp_found = "true";
- }
- if (detection.return == temp_found) {
- //console.log(text);
- //console.log(detection.string," > Found ocr");
- temp_profile.found += 1;
- }
- }
- })
- .catch(error => {
- verbose && console.log(error.message);
- })
- tmpobj.removeCallback();
- } else if (detection.type == "normal" && source != "") {
- if (source.toLowerCase().includes(detection.string.replace("{username}", username).toLowerCase())) {
- temp_found = "true";
- }
- if (detection.return == temp_found) {
- //console.log(detection.string," > normal");
- temp_profile.found += 1
- }
- } else if (detection.type == "advanced" && text_only != "") {
- if (text_only.toLowerCase().includes(detection.string.replace("{username}", username).toLowerCase())) {
- temp_found = "true";
- }
- if (detection.return == temp_found) {
- //console.log(detection.string," > normal");
- temp_profile.found += 1
- }
- }
-
- } catch (err) {
-
- }
- }));
- }
- if (temp_profile.found > 0 || temp_profile.image != "") {
- temp_profile.text = sanitizeHtml(text_only);
- temp_profile.title = sanitizeHtml(title);
- temp_profile.rate = "%" + ((temp_profile.found / site.detections.length) * 100).toFixed(2);
- temp_profile.link = site.url.replace("{username}", username);
- temp_profile.type = site.type
- resolve(temp_profile);
- } else {
- resolve(undefined)
- }
- } catch (err) {
- if (driver !== undefined) {
- try {
- await driver.quit()
- } catch (err) {
- verbose && console.log("Driver Session Issue")
- }
- }
- resolve(undefined)
- }
- });
-}
-
-async function find_username_normal(req) {
-
- var functions = [];
- var detections_result = [];
-
- async function find_username_site(uuid, username, options, site, body) {
- try {
- log_to_file_queue(uuid, "[Checking] " + get_site_from_url(site.url))
- var detections_count = 0;
- var source = body;
- var text_only = "unavailable";
- var title = "unavailable";
- var temp_profile = {
- "found": 0,
- "image": "",
- "link": "",
- "rate": "",
- "title": "",
- "text": "",
- "type": ""
- };
-
- await Promise.all(site.detections.map(async detection => {
- var temp_found = "false";
- if (detection.type == "normal" && options.includes("FindUserProfilesFast") && source != "") {
- detections_count += 1
- if (source.toLowerCase().includes(detection.string.replace("{username}", username).toLowerCase())) {
- temp_found = "true";
- }
- if (detection.return == temp_found) {
- //console.log(detection.string, " > normal");
- temp_profile.found += 1
- }
- }
- }));
- if (temp_profile.found > 0 && detections_count != 0) {
- temp_profile.text = sanitizeHtml(htmlToText(body, {
- wordwrap: false,
- hideLinkHrefIfSameAsText: true,
- ignoreHref: true,
- ignoreImage: true
- }));
- if (temp_profile.text == "") {
- temp_profile.text = "unavailable"
- }
-
- try {
- var $ = cheerio.load(body);
- title = sanitizeHtml($("title").text())
- if (title.length == 0) {
- title = "unavailable"
- }
- } catch (err) {
- verbose && console.log(err);
- }
-
- temp_profile.title = title;
- temp_profile.rate = "%" + ((temp_profile["found"] / detections_count) * 100).toFixed(2);
- temp_profile.link = site.url.replace("{username}", username);
- temp_profile.type = site.type
- return Promise.resolve(temp_profile);
- }
- return Promise.resolve();
- } catch (err) {
- return Promise.reject();
- }
- }
-
- async function find_username_sites(uuid, username, options, parsed_sites) {
-
- await parsed_sites.forEach(site => {
- if ("status" in site) {
- if (site.status == "bad") {
- return;
- }
- }
- if (site.selected == "true" && site.detections.length > 0) {
- functions.push(function(callback) {
- var request = https.get(site.url.replace("{username}", username), header_options, function(res) {
- var body = ""
- res.on("data", function(chunk) {
- body += chunk;
- });
- res.on("end", async function() {
- var results = await find_username_site(uuid, username, options, site, body);
- detections_result.push(results);
- callback(null, "Done!");
- });
- });
- request.on('error', function(e) {
- callback(null, "Done!");
- });
- request.on('socket', function(socket) {
- var timeout = (site.timeout != 0) ? site.timeout * 1000 : 5000;
- socket.setTimeout(timeout, function() {
- request.abort();
- });
- });
- });
- }
- });
- }
-
- await find_username_sites(req.body.uuid, req.body.string, req.body.option, parsed_sites);
- await async.parallelLimit(functions, 100);
- return detections_result.filter(item => item !== undefined);
-}
-
-async function find_username_advanced_2(username, options) {
-
- var detections_result = [];
-
- async function find_username_site(username, options, driver, site) {
- try {
- if ("status" in site) {
- if (site.status == "bad") {
- return Promise.resolve();
- }
- }
- if (site.selected == "true" && site.detections.length > 0 || site.selected == "true" && options.includes("ShowUserProfilesSlow")) {
- var source = "";
- var data = "";
- var text_only = "unavailable";
- var title = "unavailable";
- var temp_profile = {
- "found": 0,
- "image": "",
- "link": "",
- rate: "",
- title: "",
- text: ""
- };
- var link = site.url.replace("{username}", username);
- await driver.get(link);;
- source = await driver.getPageSource();
- data = await driver.takeScreenshot();
- title = await driver.getTitle();
- text_only = await driver.findElement(By.tagName("body")).getText();
- if (options.includes("ShowUserProfilesSlow")) {
- temp_profile["image"] = "data:image/png;base64,{image}".replace("{image}", data);
- }
- if (site.selected == "true" && site.detections.length > 0 && options.includes("FindUserProfilesSlow")) {
- await Promise.all(site.detections.map(async detection => {
- if ("status" in detection) {
- if (detection.status == "bad") {
- return;
- }
- }
- var temp_found = "false"
- if (detection.type == "ocr" && data != "") {
- tmpobj = tmp.fileSync();
- fs.writeFileSync(tmpobj.name, Buffer.from(data, "base64"));
- await tesseract.recognize(tmpobj.name, {
- lang: "eng",
- oem: 1,
- psm: 3,
- })
- .then(text => {
- text = text.replace(/[^A-Za-z0-9]/gi, "");
- detection.string = detection.string.replace(/[^A-Za-z0-9]/gi, "");
- if (text != "") {
- if (text.toLowerCase().includes(detection.string.toLowerCase())) {
- temp_found = "true";
- }
- if (detection.return == temp_found) {
- //console.log(text);
- //console.log(detection.string," > Found ocr");
- temp_profile.found += 1;
- }
- }
- })
- .catch(error => {
- verbose && console.log(error.message);
- })
- tmpobj.removeCallback();
- } else if (detection.type == "normal" && source != "") {
- if (source.toLowerCase().includes(detection.string.replace("{username}", username).toLowerCase())) {
- temp_found = "true";
- }
- if (detection.return == temp_found) {
- //console.log(detection.string," > normal");
- temp_profile.found += 1
- }
- }
- }));
- }
- if (temp_profile.found > 0 || temp_profile.image != "") {
- temp_profile.text = sanitizeHtml(text_only);
- temp_profile.title = sanitizeHtml(title);
- temp_profile.rate = "%" + ((temp_profile.found / site.detections.length) * 100).toFixed(2);
- temp_profile.link = site.url.replace("{username}", username);
- return Promise.resolve(temp_profile);
- }
- }
- return Promise.resolve();
- } catch (err) {
- return Promise.reject();
- }
- }
-
- async function find_username_sites(username, options, driver, parsed_sites) {
- for (var site of parsed_sites) {
- var result = await find_username_site(username, options, driver, site);
- detections_result.push(result);
- }
-
- return detections_result;
- }
-
- let driver = new Builder()
- .forBrowser("firefox")
- .setFirefoxOptions(new firefox.Options().headless().windowSize({
- width: 640,
- height: 480
- }))
- .build();
-
- var timeouts = {
- implicit: 0,
- pageLoad: 10000,
- script: 10000
- };
-
- await driver.manage().setTimeouts(timeouts);
- var results = await find_username_sites(username, options, driver, parsed_sites);
- await driver.quit();
- return results.filter(item => item !== undefined);
-}
-
-async function custom_search_ouputs(req) {
- var possible_parameters = ['user', 'profile', 'account']
- const time = new Date();
- const functions = [];
- possible_parameters.forEach((key) => {
- functions.push(custom_search_ouputs_website.bind(null, req.body.uuid, req.body.string, key));
- });
- const results = await async.parallelLimit(functions, 6);
- verbose && console.log(`Total time ${new Date() - time}`);
- var merged = [].concat.apply([], results.filter(item => item !== undefined));
- return merged
-}
-
-async function custom_search_ouputs_website(uuid, name, key) {
-
- return new Promise(async (resolve, reject) => {
- try {
- var results = []
- log_to_file_queue(uuid, "[Custom Search Using] " + key)
- var url = "https://www.googleapis.com/customsearch/v1?key={0}&cx={1}&q={2}:{3}".replace("{0}", google_api_key).replace("{1}", google_api_cs).replace("{2}", key).replace("{3}", name);
- var response = await get_url_wrapper_json(url);
- if (response.data != '') {
- if ('items' in response.data) {
- response.data.items.forEach((key) => {
- results.push({
- site: get_site_from_url(key.link),
- link: key.link,
- snippet: key.snippet
- })
- });
- }
- }
- if (results.length > 0) {
- resolve(results);
- } else {
- resolve(undefined)
- }
- } catch (err) {
- resolve(undefined)
- }
- });
-}
-
app.get("/get_settings", async function(req, res, next) {
temp_list = [];
- temp_list = await Promise.all(parsed_sites.map(async (site, index) => {
+ temp_list = await Promise.all(helper.parsed_sites.map(async (site, index) => {
var temp_url = "";
if ("status" in site) {
if (site.status == "bad") {
@@ -771,8 +99,7 @@ app.get("/get_settings", async function(req, res, next) {
}
}
if (site.detections.length > 0) {
- temp_url = url.parse(site.url.replace("{username}", "nothinghere")).hostname
- temp_url = temp_url.replace("nothinghere.", "");
+ temp_url = helper.get_site_from_url(site.url)
if (temp_url != "nothinghere") {
temp_selected = "false";
if ("selected" in site) {
@@ -801,42 +128,42 @@ app.get("/get_settings", async function(req, res, next) {
return 0;
});
res.json({
- proxy: proxy,
- user_agent: header_options['headers']['User-Agent'],
- google: [google_api_key.substring(0, 10) + "******", google_api_cs.substring(0, 10) + "******"],
+ proxy: helper.proxy,
+ user_agent: helper.header_options['headers']['User-Agent'],
+ google: [helper.google_api_key.substring(0, 10) + "******", helper.google_api_cs.substring(0, 10) + "******"],
websites: temp_list
});
});
app.post("/save_settings", async function(req, res, next) {
- await parsed_sites.forEach(function(value, i) {
- parsed_sites[i].selected = "false"
+ await helper.parsed_sites.forEach(function(value, i) {
+ helper.parsed_sites[i].selected = "false"
});
if ("websites" in req.body) {
if (req.body.websites.length > 0) {
await req.body.websites.split(',').forEach(item => {
- parsed_sites[Number(item)].selected = "true";
+ helper.parsed_sites[Number(item)].selected = "true";
});
}
}
- if (req.body.google_key != google_api_key.substring(0, 10) + "******") {
- google_api_key = req.body.google_key;
+ if (req.body.google_key != helper.google_api_key.substring(0, 10) + "******") {
+ helper.google_api_key = req.body.google_key;
}
- if (req.body.google_cv != google_api_cs.substring(0, 10) + "******") {
- google_api_cs = req.body.google_cv;
+ if (req.body.google_cv != helper.google_api_cs.substring(0, 10) + "******") {
+ helper.google_api_cs = req.body.google_cv;
}
- if (req.body.user_agent != header_options['headers']['User-Agent']) {
- header_options['headers']['User-Agent'] = req.body.user_agent;
+ if (req.body.user_agent != helper.header_options['headers']['User-Agent']) {
+ helper.header_options['headers']['User-Agent'] = req.body.user_agent;
}
- if (req.body.proxy != proxy) {
- proxy = req.body.proxy;
+ if (req.body.proxy != helper.proxy) {
+ helper.proxy = req.body.proxy;
}
- if (proxy != "") {
- header_options['agent'] = HttpsProxyAgent(proxy)
+ if (helper.proxy != "") {
+ helper.header_options['agent'] = HttpsProxyAgent(helper.proxy)
} else {
- if ('agent' in header_options) {
- delete header_options['agent'];
+ if ('agent' in helper.header_options) {
+ delete helper.header_options['agent'];
}
}
@@ -857,323 +184,10 @@ app.get("/generate", async function(req, res, next) {
res.json({
combinations: list_of_combinations
});
-
});
-async function find_origins(req) {
- var found = []
- for (key in parsed_names_origins) {
- for (name in parsed_names_origins[key]['boy']) {
- if (req.body.string.includes(parsed_names_origins[key]['boy'][name])) {
- found.push({
- "name": parsed_names_origins[key]['boy'][name],
- "origin": key,
- "gender": "boy",
- "matched": parsed_names_origins[key]['boy'][name],
- "similar": ""
- })
- } else {
- var similarity = stringSimilarity.compareTwoStrings(req.body.string, parsed_names_origins[key]['boy'][name]);
- if (similarity > 0.7) {
- found.push({
- "name": req.body.string,
- "origin": key,
- "gender": "boy",
- "matched": "",
- "similar": parsed_names_origins[key]['boy'][name]
- })
- }
- }
- }
- for (name in parsed_names_origins[key]['girl']) {
- if (req.body.string.includes(parsed_names_origins[key]['girl'][name])) {
- found.push({
- "name": parsed_names_origins[key]['girl'][name],
- "origin": key,
- "gender": "girl",
- "matched": parsed_names_origins[key]['girl'][name],
- "similar": ""
- })
- } else {
- var similarity = stringSimilarity.compareTwoStrings(req.body.string, parsed_names_origins[key]['girl'][name]);
- if (similarity > 0.7) {
- found.push({
- "name": req.body.string,
- "origin": key,
- "gender": "girl",
- "matched": "",
- "similar": parsed_names_origins[key]['girl'][name]
- })
- }
- }
- }
- for (name in parsed_names_origins[key]['uni']) {
- if (req.body.string.includes(parsed_names_origins[key]['uni'][name])) {
- found.push({
- "name": parsed_names_origins[key]['uni'][name],
- "origin": key,
- "gender": "uni",
- "matched": parsed_names_origins[key]['uni'][name],
- "similar": ""
- })
- } else {
- var similarity = stringSimilarity.compareTwoStrings(req.body.string, parsed_names_origins[key]['uni'][name]);
- if (similarity > 0.7) {
- found.push({
- "name": req.body.string,
- "origin": key,
- "gender": "uni",
- "matched": "",
- "similar": parsed_names_origins[key]['uni'][name]
- })
- }
- }
- }
- }
+app.post("/analyze_string", async function(req, res, next) {
- return found
-}
-
-async function get_words_info(all_words, words_info) {
- var temp_added = []
- for (let all_words_key of Object.keys(all_words)) {
- for (let all_words_word of all_words[all_words_key]) {
- if (!temp_added.includes(all_words_word)) {
- temp_added.push(all_words_word);
- var temp_words_info = {
- "word": all_words_word,
- "text": "",
- "results": []
- }
- try {
- var url1 = "https://api.duckduckgo.com/?q={0}&format=json&pretty=1&no_html=1&skip_disambig=1".replace("{0}", all_words_word);
- var url2 = "https://api.duckduckgo.com/?q={0}&format=json&pretty=1".replace("{0}", all_words_word);
- var response1 = await get_url_wrapper_json(url1);
- var response2 = await get_url_wrapper_json(url2);
- if (response2.data != '') {
- if ("RelatedTopics" in response2.data) {
- if (response2.data.RelatedTopics.length > 0) {
- if (response2.data != '') {
- if ("AbstractText" in response1.data && response1.data.AbstractText != "") {
- temp_words_info.text = response1.data.AbstractText;
- } else if ("Abstract" in response1.data && response1.data.AbstractText != "") {
- temp_words_info.text = response1.data.Abstract;
- } else {
- temp_words_info.text = "unknown";
- }
- }
- response2.data.RelatedTopics.forEach(function(item) {
- if ("Name" in item) {
- item.Topics.forEach(function(topic) {
- temp_words_info.results.push({
- "type": item.Name,
- "text": topic.Text,
- "url": topic.FirstURL
- });
- });
- } else {
- temp_words_info.results.push({
- "type": "Related",
- "text": item.Text,
- "url": item.FirstURL
- });
- }
- });
- }
- }
- }
-
- if (temp_words_info.results.length > 0) {
- words_info.push(temp_words_info);
- }
- } catch (error) {
- verbose && console.log(error);
- }
- }
- }
- }
-}
-
-async function check_engines(req, info) {
- try {
- if (google_api_key == "" || google_api_cs == "") {
- return
- }
- var url = "https://www.googleapis.com/customsearch/v1?key={0}&cx={1}&q={2}".replace("{0}", google_api_key).replace("{1}", google_api_cs).replace("{2}", req.body.string);
- var response = await get_url_wrapper_json(url);
- if (response.data != '') {
- try {
- info.original = response.data.queries.request[0].searchTerms
- } catch (e) {}
- try {
- info.corrected = response.data.spelling.correctedQuery
- } catch (e) {}
- try {
- info.total = response.data.searchInformation.totalResults
- } catch (e) {}
- try {
- response.data.items.forEach(function(item) {
- info["items"].push({
- "title": item.title,
- "snippet": item.snippet
- });
- });
- } catch (e) {}
- try {
- if (info.total == 0 && info.corrected != "") {
- info.checking = info.original + " [Error]
Try this: " + info.corrected;
- } else if (info.total > 0 && info.corrected != "") {
- info.checking = info.original + " [Good]
Suggested word: " + info.corrected + "
Total lookups: " + info.total;
- } else if (info.total > 0 && info.corrected == "") {
- info.checking = info.original + " [Good]
Total lookups: " + info.total;
- } else {
- info.checking = "Using " + info.original + " with no lookups";
- }
- } catch (e) {}
- }
- } catch (error) {
- verbose && console.log(error);
- }
-}
-
-function most_common1(all_words, temp_words) {
- var temp_list = []
- Object.keys(all_words).forEach(function(key) {
- all_words[key].forEach(function(item) {
- if (!temp_list.includes(item) && item.length > 1) {
- temp_list.push(item);
- var temp = findWord(item);
- if (Object.keys(temp).length != 0) {
- var languages = Object.keys(temp).map(function(key) {
- return [key, temp[key]];
- });
- languages.sort(function(first, second) {
- return second[1] - first[1]
- }).reverse();
- temp_words.push({
- "word": item,
- "languages": languages.map(e => e.join(":")).join(" ")
- });
- }
- }
- });
- });
-}
-
-async function most_common(all_words, temp_words) {
- var temp_list = []
- Object.keys(all_words).forEach(function(key) {
- all_words[key].forEach(function(item) {
- if (!temp_list.includes(item) && item.length > 1) {
- temp_list.push(item);
- var temp = findWord(item);
- if (Object.keys(temp).length != 0) {
- var languages = Object.keys(temp).map(function(key) {
- return [key, temp[key]];
- });
- languages.sort(function(first, second) {
- return second[1] - first[1]
- }).reverse();
- temp_words.push({
- "word": item,
- "languages": languages.map(e => e[0]).join(", ")
- });
- }
- }
- });
- });
-}
-
-function find_other(all_words) {
- var words = WordsNinja.splitSentence(req.body.string);
-
- words.forEach(function(word) {
- var value = false
- Object.keys(all_words).forEach(function(key) {
- if (all_words[key].includes(word)) {
- value = true
- }
- });
-
- if (!value && !all_words.maybe.includes(word)) {
- all_words.maybe.push(word);
- }
- });
-}
-
-function remove_word(str, sub_string) {
- part1 = str.substring(0, str.indexOf(sub_string));
- part2 = str.substring(str.indexOf(sub_string) + sub_string.length, str.length);
- temp = (part1 + part2).replace(/[ \[\]:"\\|,.<>\/?~`!@#$%^&*()_+\-={};"]/gi, "");
- return temp;
-}
-
-async function analyze_name(req, all_words) {
- log_to_file_queue(req.body.uuid, "[Starting] String analysis")
- temp_rr_names = []
- string_to_check = req.body.string
- parsed_json.prefix.forEach(function(item, index) {
- if (string_to_check.indexOf(item) == 0 && !all_words.prefix.includes(item)) {
- all_words.prefix.push(item);
- temp = remove_word(string_to_check, item);
- if (temp !== null && temp !== "" && !all_words.unknown.includes(temp) && !all_words.maybe.includes(temp) && temp.length > 1) {
- all_words.unknown.push(temp);
- }
- }
- });
- parsed_json.m_names.forEach(function(item, index) {
- if (string_to_check.indexOf(item) >= 0 && !all_words.name.includes(item)) {
- all_words.name.push(item);
- temp = remove_word(string_to_check, item);
- if (temp !== null && temp !== "" && !all_words.unknown.includes(temp) && !all_words.maybe.includes(temp) && temp.length > 1) {
- all_words.unknown.push(temp);
- }
- }
- });
- parsed_json.f_names.forEach(function(item, index) {
- if (string_to_check.indexOf(item) >= 0 && !all_words.name.includes(item)) {
- all_words.name.push(item);
- temp = remove_word(string_to_check, item);
- if (temp !== null && temp !== "" && !all_words.unknown.includes(temp) && !all_words.maybe.includes(temp) && temp.length > 1) {
- all_words.unknown.push(temp);
- }
- }
- });
-
- all_words.prefix.forEach(function(h_item, index) {
- all_words.unknown.forEach(function(r_item, index) {
- if (r_item.indexOf(h_item) == 0) {
- temp = remove_word(r_item, h_item);
- if (temp !== null && temp !== "" && !temp_rr_names.includes(temp) && !all_words.maybe.includes(temp) && temp.length > 1) {
- temp_rr_names.push(temp);
- }
- }
- });
- });
-
- var temp_r_concat = all_words.unknown.concat(temp_rr_names.filter((item) => all_words.unknown.indexOf(item) < 0));
-
- all_words.unknown = temp_r_concat
- temp_rr_names = []
-
- all_words.number.forEach(function(n_item, index) {
- all_words.unknown.forEach(function(r_item, index) {
- if (r_item.indexOf(n_item) >= 0) {
- temp = remove_word(r_item, n_item);
- if (temp !== null && temp !== "" && !temp_rr_names.includes(temp) && !all_words.maybe.includes(temp) && temp.length > 1) {
- temp_rr_names.push(temp);
- }
- }
- });
- });
-
- var temp_r_concat = all_words.unknown.concat(temp_rr_names.filter((item) => all_words.unknown.indexOf(item) < 0));
- all_words.unknown = temp_r_concat
- log_to_file_queue(req.body.uuid, "[Done] String analysis")
-}
-
-app.post("/url", async function(req, res, next) {
- await WordsNinja.loadDictionary();
var info = {
"items": [],
"original": "",
@@ -1210,14 +224,14 @@ app.post("/url", async function(req, res, next) {
} else {
req.body.uuid = req.body.uuid.replace(/[^a-zA-Z0-9\-]+/g, '');
if (req.body.option.includes("FindUserProfilesSpecial")) {
- log_to_file_queue(req.body.uuid, "[Starting] Checking user profiles special")
- user_info_special.data = await find_username_special(req);
- log_to_file_queue(req.body.uuid, "[Done] Checking user profiles special")
+ helper.log_to_file_queue(req.body.uuid, "[Starting] Checking user profiles special")
+ user_info_special.data = await special.find_username_special(req);
+ helper.log_to_file_queue(req.body.uuid, "[Done] Checking user profiles special")
}
if (req.body.option.includes("FindUserProfilesFast")) {
- log_to_file_queue(req.body.uuid, "[Starting] Checking user profiles normal")
- user_info_advanced.data = await find_username_normal(req);
- log_to_file_queue(req.body.uuid, "[Done] Checking user profiles normal")
+ helper.log_to_file_queue(req.body.uuid, "[Starting] Checking user profiles normal")
+ user_info_advanced.data = await fastscan.find_username_normal(req);
+ helper.log_to_file_queue(req.body.uuid, "[Done] Checking user profiles normal")
}
if (req.body.option.includes("FindUserProfilesSlow") || req.body.option.includes("ShowUserProfilesSlow")) {
if (!req.body.option.includes("FindUserProfilesSlow")) {
@@ -1225,73 +239,51 @@ app.post("/url", async function(req, res, next) {
} else if (!req.body.option.includes("ShowUserProfilesSlow")) {
user_info_normal.type = "noshow"
}
- log_to_file_queue(req.body.uuid, "[Starting] Checking user profiles advanced")
- user_info_normal.data = await find_username_advanced(req);
- log_to_file_queue(req.body.uuid, "[Done] Checking user profiles advanced")
+ helper.log_to_file_queue(req.body.uuid, "[Starting] Checking user profiles advanced")
+ user_info_normal.data = await slowscan.find_username_advanced(req);
+ helper.log_to_file_queue(req.body.uuid, "[Done] Checking user profiles advanced")
}
if (req.body.option.includes("LookUps")) {
- log_to_file_queue(req.body.uuid, "[Starting] Lookup")
- await check_engines(req, info);
- log_to_file_queue(req.body.uuid, "[Done] Lookup")
+ helper.log_to_file_queue(req.body.uuid, "[Starting] Lookup")
+ await externalapis.check_engines(req, info);
+ helper.log_to_file_queue(req.body.uuid, "[Done] Lookup")
}
if (req.body.option.includes("CustomSearch")) {
- log_to_file_queue(req.body.uuid, "[Starting] Custom Search")
- custom_search = await custom_search_ouputs(req);
- log_to_file_queue(req.body.uuid, "[Done] Custom Search")
+ helper.log_to_file_queue(req.body.uuid, "[Starting] Custom Search")
+ custom_search = await externalapis.custom_search_ouputs(req);
+ helper.log_to_file_queue(req.body.uuid, "[Done] Custom Search")
+ }
+ if (req.body.option.includes("FindOrigins")) {
+ helper.log_to_file_queue(req.body.uuid, "[Starting] Finding Origins")
+ names_origins = await nameanalysis.find_origins(req);
+ helper.log_to_file_queue(req.body.uuid, "[Done] Finding Origins")
}
if (req.body.option.includes("SplitWordsByUpperCase")) {
- try {
- req.body.string.match(/[A-Z][a-z]+/g).forEach((item) => {
- if (item.length > 1 && !all_words.unknown.includes(item) && !all_words.maybe.includes(item)) {
- all_words.unknown.push(item.toLowerCase());
- }
- });
- log_to_file_queue(req.body.uuid, "[Done] Split by UpperCase")
- } catch (err) {}
+ helper.log_to_file_queue(req.body.uuid, "[Starting] Split by UpperCase")
+ await stringanalysis.split_upper_case(req,all_words)
+ helper.log_to_file_queue(req.body.uuid, "[Done] Split by UpperCase")
}
-
if (req.body.option.includes("SplitWordsByAlphabet")) {
- try {
- req.body.string.match(/[A-Za-z]+/g).forEach((item) => {
- if (item.length > 1 && !all_words.unknown.includes(item) && !all_words.maybe.includes(item)) {
- all_words.unknown.push(item.toLowerCase());
- }
- });
- log_to_file_queue(req.body.uuid, "[Done] Split by Alphabet")
- } catch (err) {}
+ helper.log_to_file_queue(req.body.uuid, "[Starting] Split by Alphabet")
+ await stringanalysis.split_alphabet_case(req,all_words)
+ helper.log_to_file_queue(req.body.uuid, "[Done] Split by Alphabet")
}
-
- if (req.body.option.includes("FindOrigins")) {
- log_to_file_queue(req.body.uuid, "[Starting] Finding Origins")
- names_origins = await find_origins(req);
- log_to_file_queue(req.body.uuid, "[Done] Finding Origins")
+ if (req.body.option.includes("FindSymbols")) {
+ helper.log_to_file_queue(req.body.uuid, "[Starting] Finding Symbols")
+ await stringanalysis.find_symbols(req,all_words)
+ helper.log_to_file_queue(req.body.uuid, "[Done] Finding Symbols")
+ }
+ if (req.body.option.includes("FindNumbers")) {
+ helper.log_to_file_queue(req.body.uuid, "[Starting] Finding Numbers")
+ await stringanalysis.find_numbers(req,all_words)
+ helper.log_to_file_queue(req.body.uuid, "[Done] Finding Numbers")
}
-
req.body.string = req.body.string.toLowerCase();
if (req.body.option.includes("ConvertNumbers")) {
- numbers_to_letters = {
- "4": "a",
- "8": "b",
- "3": "e",
- "1": "l",
- "0": "o",
- "5": "s",
- "7": "t",
- "2": "z"
- }
-
- temp_value = ""
- for (i = 0; i < req.body.string.length; i++) {
- _temp = numbers_to_letters[req.body.string.charAt(i)]
- if (_temp != undefined) {
- temp_value += numbers_to_letters[req.body.string.charAt(i)];
- } else {
- temp_value += req.body.string.charAt(i);
- }
- }
- req.body.string = temp_value
- log_to_file_queue(req.body.uuid, "[Done] Convert numbers to letters")
+ helper.log_to_file_queue(req.body.uuid, "[Starting] Convert Numbers")
+ await stringanalysis.convert_numbers(req,all_words)
+ helper.log_to_file_queue(req.body.uuid, "[Done] Convert Numbers")
}
if (req.body.option.includes("LookUps") ||
@@ -1302,47 +294,20 @@ app.post("/url", async function(req, res, next) {
req.body.option.includes("FindSymbols") ||
req.body.option.includes("FindNumbers") ||
req.body.option.includes("ConvertNumbers")) {
- if (req.body.option.includes("FindNumbers")) {
- try {
- req.body.string.match(/(\d+)/g).forEach((item) => {
- if (!all_words.number.includes(item)) {
- all_words.number.push(item);
- }
- });
- } catch (err) {}
- }
- if (req.body.option.includes("FindSymbols")) {
- try {
- req.body.string.match(/[ \[\]:"\\|,.<>\/?~`!@#$%^&*()_+\-={};']/gi).forEach((item) => {
- if (item !== " " && !all_words.symbol.includes(item)) {
- all_words.symbol.push(item);
- }
- });
- } catch (err) {}
- }
+ await stringanalysis.get_maybe_words(req, all_words)
+ await stringanalysis.analyze_string(req, all_words);
- if (req.body.option.includes("SplitUpperCase")) {
- req.body.string = req.body.string.replace(/([A-Z]+)/g, " $1");
- if (req.body.string.startsWith(" ")) {
- req.body.string = req.body.string.substring(1);
- }
- }
- all_words.maybe = WordsNinja.splitSentence(req.body.string).filter(function(elem, index, self) {
- return index === self.indexOf(elem);
- }).filter(word => word.length > 1);
- await analyze_name(req, all_words);
- //find_other(all_words)
Object.keys(all_words).forEach((key) => (all_words[key].length == 0) && delete all_words[key]);
if (req.body.option.includes("MostCommon")) {
- await most_common(all_words, temp_words);
+ await stringanalysis.most_common(all_words, temp_words);
}
if (req.body.option.includes("WordInfo")) {
- await get_words_info(all_words, words_info);
+ await externalapis.get_words_info(all_words, words_info);
}
} else if (req.body.option.includes("NormalAnalysis@@")) {
- var maybe_words = WordsNinja.splitSentence(req.body.string);
+ //var maybe_words = WordsNinja.splitSentence(req.body.string);
all_words.maybe = maybe_words.filter(function(elem, index, self) {
return index === self.indexOf(elem);
});
@@ -1373,84 +338,101 @@ app.post("/url", async function(req, res, next) {
});
app.use((err, req, res, next) => {
- verbose && console.log(" --- Global Error ---")
- verbose && console.log(pe.render(err));
+ helper.verbose && console.log(" --- Global Error ---")
+ helper.verbose && console.log(pe.render(err));
res.json("Error");
});
process.on('uncaughtException', function(err) {
- verbose && console.log(" --- Uncaught Error ---")
- verbose && console.log(pe.render(err));
+ helper.verbose && console.log(" --- Uncaught Error ---")
+ helper.verbose && console.log(pe.render(err));
})
process.on('unhandledRejection', function(err) {
- verbose && console.log(" --- Uncaught Rejection ---")
- verbose && console.log(pe.render(err));
+ helper.verbose && console.log(" --- Uncaught Rejection ---")
+ helper.verbose && console.log(pe.render(err));
})
const server_host = '0.0.0.0';
const server_port = process.env.PORT || 9005;
-
-async function check_user_cli(username, websites) {
+async function check_user_cli(argv) {
var ret = []
var random_string = Math.random().toString(36).substring(2);
var req = {
'body': {
'uuid': random_string,
- 'string': username,
- 'option': 'FindUserProfilesFast'
+ 'string': argv.username,
+ 'option': 'FindUserProfilesFast,' + argv.output
}
}
- await parsed_sites.forEach(async function(value, i) {
- parsed_sites[i].selected = "false"
- if (websites.length > 0) {
- await websites.split(' ').forEach(item => {
- if (parsed_sites[i].url.toLowerCase().includes(item.toLowerCase())) {
- parsed_sites[i].selected = "true"
+ await helper.parsed_sites.forEach(async function(value, i) {
+ helper.parsed_sites[i].selected = "false"
+ if (argv.websites.length > 0) {
+ await argv.websites.split(' ').forEach(item => {
+ if (helper.parsed_sites[i].url.toLowerCase().includes(item.toLowerCase())) {
+ helper.parsed_sites[i].selected = "true"
}
});
}
});
- ret = await find_username_normal(req)
+
+ ret = await fastscan.find_username_normal(req)
if (typeof ret === 'undefined' || ret === undefined || ret.length == 0) {
- log_to_file_queue(req.body.uuid, 'User does not exist (try FindUserProfilesSlow or FindUserProfilesSpecial)');
+ helper.log_to_file_queue(req.body.uuid, 'User does not exist (try FindUserProfilesSlow or FindUserProfilesSpecial)');
} else {
+ var temp_detected = []
await ret.forEach(item => {
- delete item['title']
- delete item['image']
- delete item['text']
- item['link'] = get_site_from_url(item['link'])
- log_to_file_queue(req.body.uuid, item);
+ var temp_keys = {"found": 0,"link": "","rate": "","title": "","text": "","image":""};
+ if (argv.options == "")
+ {
+ delete item['text']
+ delete item['image']
+ }
+ else{
+ for (var key in temp_keys) {
+ if (!argv.options.includes(key)){
+ delete item[key]
+ }
+ }
+ }
+
+ temp_detected.push(item)
+
+ if (argv.output == "pretty"){
+ helper.log_to_file_queue(req.body.uuid, item);
+ }
+ else if (argv.output == ""){
+ helper.log_to_file_queue(req.body.uuid, item);
+ }
});
+
+ if (argv.output == "json"){
+ console.log(JSON.stringify(temp_detected, null, 2))
+ }
}
};
async function list_all_websites() {
var temp_arr = []
- await parsed_sites.forEach(item => {
- temp_arr.push(get_site_from_url(item.url))
+ await helper.parsed_sites.forEach(item => {
+ temp_arr.push(helper.get_site_from_url(item.url))
});
+
console.log('[Listing] Available websites\n' + temp_arr.join('\n'))
}
-if ('grid' in argv) {
- grid_url = argv.grid
+if (argv.grid != "") {
+ helper.grid_url = argv.grid
}
-if ('cli' in argv) {
- if ('list' in argv) {
+if (argv.cli) {
+ if (argv.list) {
list_all_websites();
- } else if ('mode' in argv) {
- if (argv.mode == 'fast') {
- if ('user' in argv && 'website' in argv) {
- if (argv.user != "" && argv.website != "") {
- check_user_cli(argv.user, argv.website)
- } else {
- console.log("user or website is empty, use -h for help")
- }
- }
+ } else if (argv.mode == "fast") {
+ if (argv.usernmae != "" && argv.websites != "") {
+ check_user_cli(argv)
}
}
} else {
diff --git a/info b/info
index 5e76488..c8c00ba 100644
--- a/info
+++ b/info
@@ -1,9 +1,9 @@
-{"version":"2021.V.2.09",
+{"version":"2021.V.2.10",
"build":"pass",
"test":"pass",
"grid_test":"pass",
"websites":"342",
"detections":"939",
- "special":"2",
+ "special":"3",
"awaiting_verification":"22",
- "auto_testing":"606e2f07-cf0d-4f05-98e0-bfd6ed204a1a"}
+ "auto_testing":"bb581703-6333-4d78-bb30-e20a9e846145"}
diff --git a/modules/externalapis.js b/modules/externalapis.js
new file mode 100644
index 0000000..51408ac
--- /dev/null
+++ b/modules/externalapis.js
@@ -0,0 +1,153 @@
+var helper = require("./helper.js")
+var async = require("async");
+
+async function get_words_info(all_words, words_info) {
+ var temp_added = []
+ for (let all_words_key of Object.keys(all_words)) {
+ for (let all_words_word of all_words[all_words_key]) {
+ if (!temp_added.includes(all_words_word)) {
+ temp_added.push(all_words_word);
+ var temp_words_info = {
+ "word": all_words_word,
+ "text": "",
+ "results": []
+ }
+ try {
+ var url1 = "https://api.duckduckgo.com/?q={0}&format=json&pretty=1&no_html=1&skip_disambig=1".replace("{0}", all_words_word);
+ var url2 = "https://api.duckduckgo.com/?q={0}&format=json&pretty=1".replace("{0}", all_words_word);
+ var response1 = await helper.get_url_wrapper_json(url1);
+ var response2 = await helper.get_url_wrapper_json(url2);
+ if (response2.data != '') {
+ if ("RelatedTopics" in response2.data) {
+ if (response2.data.RelatedTopics.length > 0) {
+ if (response2.data != '') {
+ if ("AbstractText" in response1.data && response1.data.AbstractText != "") {
+ temp_words_info.text = response1.data.AbstractText;
+ } else if ("Abstract" in response1.data && response1.data.AbstractText != "") {
+ temp_words_info.text = response1.data.Abstract;
+ } else {
+ temp_words_info.text = "unknown";
+ }
+ }
+ response2.data.RelatedTopics.forEach(function(item) {
+ if ("Name" in item) {
+ item.Topics.forEach(function(topic) {
+ temp_words_info.results.push({
+ "type": item.Name,
+ "text": topic.Text,
+ "url": topic.FirstURL
+ });
+ });
+ } else {
+ temp_words_info.results.push({
+ "type": "Related",
+ "text": item.Text,
+ "url": item.FirstURL
+ });
+ }
+ });
+ }
+ }
+ }
+
+ if (temp_words_info.results.length > 0) {
+ words_info.push(temp_words_info);
+ }
+ } catch (error) {
+ helper.verbose && console.log(error);
+ }
+ }
+ }
+ }
+}
+
+async function check_engines(req, info) {
+ try {
+ if (helper.google_api_key == "" || helper.google_api_cs == "") {
+ return
+ }
+ var url = "https://www.googleapis.com/customsearch/v1?key={0}&cx={1}&q={2}".replace("{0}", helper.google_api_key).replace("{1}", helper.google_api_cs).replace("{2}", req.body.string);
+ var response = await helper.get_url_wrapper_json(url);
+ if (response.data != '') {
+ try {
+ info.original = response.data.queries.request[0].searchTerms
+ } catch (e) {}
+ try {
+ info.corrected = response.data.spelling.correctedQuery
+ } catch (e) {}
+ try {
+ info.total = response.data.searchInformation.totalResults
+ } catch (e) {}
+ try {
+ response.data.items.forEach(function(item) {
+ info["items"].push({
+ "title": item.title,
+ "snippet": item.snippet
+ });
+ });
+ } catch (e) {}
+ try {
+ if (info.total == 0 && info.corrected != "") {
+ info.checking = info.original + " [Error]
Try this: " + info.corrected;
+ } else if (info.total > 0 && info.corrected != "") {
+ info.checking = info.original + " [Good]
Suggested word: " + info.corrected + "
Total lookups: " + info.total;
+ } else if (info.total > 0 && info.corrected == "") {
+ info.checking = info.original + " [Good]
Total lookups: " + info.total;
+ } else {
+ info.checking = "Using " + info.original + " with no lookups";
+ }
+ } catch (e) {}
+ }
+ } catch (error) {
+ helper.verbose && console.log(error);
+ }
+}
+
+async function custom_search_ouputs(req) {
+ var possible_parameters = ['user', 'profile', 'account']
+ const time = new Date();
+ const functions = [];
+ possible_parameters.forEach((key) => {
+ functions.push(custom_search_ouputs_website.bind(null, req.body.uuid, req.body.string, key));
+ });
+ const results = await async.parallelLimit(functions, 6);
+ helper.verbose && console.log(`Total time ${new Date() - time}`);
+ var merged = [].concat.apply([], results.filter(item => item !== undefined));
+ return merged
+}
+
+async function custom_search_ouputs_website(uuid, name, key) {
+
+ return new Promise(async (resolve, reject) => {
+ try {
+ var results = []
+ helper.log_to_file_queue(uuid, "[Custom Search Using] " + key)
+ var url = "https://www.googleapis.com/customsearch/v1?key={0}&cx={1}&q={2}:{3}".replace("{0}", helper.google_api_key).replace("{1}", helper.google_api_cs).replace("{2}", key).replace("{3}", name);
+ var response = await helper.get_url_wrapper_json(url);
+ if (response.data != '') {
+ if ('items' in response.data) {
+ response.data.items.forEach((key) => {
+ results.push({
+ site: helper.get_site_from_url(key.link),
+ link: key.link,
+ snippet: key.snippet
+ })
+ });
+ }
+ }
+ if (results.length > 0) {
+ resolve(results);
+ } else {
+ resolve(undefined)
+ }
+ } catch (err) {
+ resolve(undefined)
+ }
+ });
+}
+
+module.exports = {
+ check_engines,
+ get_words_info,
+ custom_search_ouputs
+}
diff --git a/modules/fastscan.js b/modules/fastscan.js
new file mode 100644
index 0000000..545d145
--- /dev/null
+++ b/modules/fastscan.js
@@ -0,0 +1,98 @@
+var helper = require("./helper.js")
+var async = require("async");
+var sanitizeHtml = require("sanitize-html");
+const {
+ htmlToText
+} = require('html-to-text');
+var cheerio = require('cheerio');
+
+async function find_username_normal(req) {
+ const time = new Date();
+ const functions = [];
+ await helper.parsed_sites.forEach(site => {
+ if ("status" in site) {
+ if (site.status == "bad") {
+ return Promise.resolve();
+ }
+ }
+ if (site.selected == "true" && site.detections.length > 0) {
+ functions.push(find_username_site.bind(null, req.body.uuid, req.body.string, req.body.option, site));
+ }
+ });
+ const results = await async.parallelLimit(functions, 100);
+ helper.verbose && console.log(`Total time ${new Date() - time}`);
+ return results.filter(item => item !== undefined)
+}
+
+async function find_username_site(uuid, username, options, site) {
+ return new Promise(async (resolve, reject) => {
+ try {
+ if (!options.includes('json')) {
+ helper.log_to_file_queue(uuid, "[Checking] " + helper.get_site_from_url(site.url))
+ }
+ var body = await helper.get_url_wrapper_text(site.url.replace("{username}", username));
+ var detections_count = 0;
+ var source = body;
+ var text_only = "unavailable";
+ var title = "unavailable";
+ var temp_profile = {
+ "found": 0,
+ "image": "",
+ "link": "",
+ "rate": "",
+ "title": "",
+ "text": "",
+ "type": ""
+ };
+
+ await Promise.all(site.detections.map(async detection => {
+ var temp_found = "false";
+ if (detection.type == "normal" && options.includes("FindUserProfilesFast") && source != "") {
+ detections_count += 1
+ if (source.toLowerCase().includes(detection.string.replace("{username}", username).toLowerCase())) {
+ temp_found = "true";
+ }
+ if (detection.return == temp_found) {
+ //console.log(detection.string, " > normal");
+ temp_profile.found += 1
+ }
+ }
+ }));
+ if (temp_profile.found > 0 && detections_count != 0) {
+ temp_profile.text = sanitizeHtml(htmlToText(body, {
+ wordwrap: false,
+ hideLinkHrefIfSameAsText: true,
+ ignoreHref: true,
+ ignoreImage: true
+ }));
+ if (temp_profile.text == "") {
+ temp_profile.text = "unavailable"
+ }
+
+ try {
+ var $ = cheerio.load(body);
+ title = sanitizeHtml($("title").text())
+ if (title.length == 0) {
+ title = "unavailable"
+ }
+ } catch (err) {
+ helper.verbose && console.log(err);
+ }
+
+ temp_profile.title = title;
+ temp_profile.rate = "%" + ((temp_profile["found"] / detections_count) * 100).toFixed(2);
+ temp_profile.link = site.url.replace("{username}", username);
+ temp_profile.type = site.type
+ resolve(temp_profile);
+ } else {
+ resolve(undefined)
+ }
+ } catch (err) {
+ resolve(undefined)
+ }
+ });
+}
+
+module.exports = {
+ find_username_normal
+}
diff --git a/modules/helper.js b/modules/helper.js
new file mode 100644
index 0000000..6e37f7f
--- /dev/null
+++ b/modules/helper.js
@@ -0,0 +1,112 @@
+var verbose = false
+var google_api_key = "";
+var google_api_cs = "";
+var grid_url = "";
+var proxy = ""
+
+var header_options = {
+ headers: {
+ 'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:84.0) Gecko/20100101 Firefox/84.0',
+ }
+};
+
+var https = require("follow-redirects").https;
+var fs = require("fs");
+var url = require("url");
+
+var parsed_sites = JSON.parse(fs.readFileSync("sites.json"));
+var logs_queue = Promise.resolve();
+
+function log_to_file_queue(uuid, msg) {
+ logs_queue = logs_queue.then(function() {
+ return new Promise(function(resolve) {
+ fs.appendFile("logs/" + uuid + "_log.txt", msg + "\n", function(err, data) {
+ console.log(msg)
+ resolve();
+ });
+ });
+ });
+}
+
+function get_site_from_url(_url) {
+ temp = url.parse(_url.replace("{username}", "nothinghere")).hostname
+ return temp.replace("nothinghere.", "")
+}
+
+async function get_url_wrapper_json(url, time = 5) {
+ try {
+ let http_promise = new Promise((resolve, reject) => {
+ var request = https.get(url, header_options, function(res) {
+ var body = ""
+ res.on("data", function(chunk) {
+ body += chunk;
+ });
+ res.on("end", function() {
+ resolve({
+ 'data': JSON.parse(body.toString())
+ });
+ });
+ });
+ request.on('error', function(e) {
+ reject({
+ 'data': ''
+ })
+ });
+ request.on('socket', function(socket) {
+ var timeout = (time != 0) ? time * 1000 : 5000;
+ socket.setTimeout(timeout, function() {
+ request.abort();
+ });
+ });
+ });
+ let response_body = await http_promise;
+ return response_body
+ } catch (err) {
+ verbose && console.log(err);
+ }
+}
+
+async function get_url_wrapper_text(url, time = 5) {
+ try {
+ let http_promise = new Promise((resolve, reject) => {
+ var request = https.get(url, header_options, function(res) {
+ var body = ""
+ res.on("data", function(chunk) {
+ body += chunk;
+ });
+ res.on("end", function() {
+ resolve(body);
+ });
+ });
+ request.on('error', function(e) {
+ reject({
+ 'data': ''
+ })
+ });
+ request.on('socket', function(socket) {
+ var timeout = (time != 0) ? time * 1000 : 5000;
+ socket.setTimeout(timeout, function() {
+ request.abort();
+ });
+ });
+ });
+ let response_body = await http_promise;
+ return response_body
+ } catch (err) {
+ verbose && console.log(err);
+ }
+}
+
+module.exports = {
+ parsed_sites,
+ verbose,
+ google_api_key,
+ google_api_cs,
+ grid_url,
+ header_options,
+ proxy,
+ get_site_from_url,
+ log_to_file_queue,
+ get_url_wrapper_text,
+ get_url_wrapper_json
+}
diff --git a/modules/nameanalysis.js b/modules/nameanalysis.js
new file mode 100644
index 0000000..bd0b4a9
--- /dev/null
+++ b/modules/nameanalysis.js
@@ -0,0 +1,83 @@
+var helper = require("./helper.js")
+var fs = require("fs");
+var stringSimilarity = require("string-similarity");
+
+var parsed_names_origins = JSON.parse(fs.readFileSync("names.json"));
+
+async function find_origins(req) {
+ var found = []
+ for (key in parsed_names_origins) {
+ for (name in parsed_names_origins[key]['boy']) {
+ if (req.body.string.includes(parsed_names_origins[key]['boy'][name])) {
+ found.push({
+ "name": parsed_names_origins[key]['boy'][name],
+ "origin": key,
+ "gender": "boy",
+ "matched": parsed_names_origins[key]['boy'][name],
+ "similar": ""
+ })
+ } else {
+ var similarity = stringSimilarity.compareTwoStrings(req.body.string, parsed_names_origins[key]['boy'][name]);
+ if (similarity > 0.7) {
+ found.push({
+ "name": req.body.string,
+ "origin": key,
+ "gender": "boy",
+ "matched": "",
+ "similar": parsed_names_origins[key]['boy'][name]
+ })
+ }
+ }
+ }
+ for (name in parsed_names_origins[key]['girl']) {
+ if (req.body.string.includes(parsed_names_origins[key]['girl'][name])) {
+ found.push({
+ "name": parsed_names_origins[key]['girl'][name],
+ "origin": key,
+ "gender": "girl",
+ "matched": parsed_names_origins[key]['girl'][name],
+ "similar": ""
+ })
+ } else {
+ var similarity = stringSimilarity.compareTwoStrings(req.body.string, parsed_names_origins[key]['girl'][name]);
+ if (similarity > 0.7) {
+ found.push({
+ "name": req.body.string,
+ "origin": key,
+ "gender": "girl",
+ "matched": "",
+ "similar": parsed_names_origins[key]['girl'][name]
+ })
+ }
+ }
+ }
+ for (name in parsed_names_origins[key]['uni']) {
+ if (req.body.string.includes(parsed_names_origins[key]['uni'][name])) {
+ found.push({
+ "name": parsed_names_origins[key]['uni'][name],
+ "origin": key,
+ "gender": "uni",
+ "matched": parsed_names_origins[key]['uni'][name],
+ "similar": ""
+ })
+ } else {
+ var similarity = stringSimilarity.compareTwoStrings(req.body.string, parsed_names_origins[key]['uni'][name]);
+ if (similarity > 0.7) {
+ found.push({
+ "name": req.body.string,
+ "origin": key,
+ "gender": "uni",
+ "matched": "",
+ "similar": parsed_names_origins[key]['uni'][name]
+ })
+ }
+ }
+ }
+ }
+
+ return found
+}
+
+module.exports = {
+ find_origins
+}
diff --git a/modules/slowscan.js b/modules/slowscan.js
new file mode 100644
index 0000000..8a8dca2
--- /dev/null
+++ b/modules/slowscan.js
@@ -0,0 +1,182 @@
+var helper = require("./helper.js")
+var async = require("async");
+var sanitizeHtml = require("sanitize-html");
+var tesseract = require("node-tesseract-ocr");
+var sanitizeHtml = require("sanitize-html");
+var firefox = require("selenium-webdriver/firefox");
+var {
+ Builder,
+ By,
+ Key
+} = require("selenium-webdriver");
+var tmp = require("tmp");
+var fs = require("fs");
+
+async function find_username_advanced(req) {
+ const time = new Date();
+ const functions = [];
+ helper.parsed_sites.forEach((site) => {
+ if ("status" in site) {
+ if (site.status == "bad") {
+ return Promise.resolve();
+ }
+ }
+ if (site.selected == "true" && site.detections.length > 0) {
+ functions.push(find_username_site.bind(null, req.body.uuid, req.body.string, req.body.option, site));
+ }
+ });
+ const results = await async.parallelLimit(functions, 8);
+ helper.verbose && console.log(`Total time ${new Date() - time}`);
+ return results.filter(item => item !== undefined)
+}
+
+async function find_username_site(uuid, username, options, site) {
+ return new Promise(async (resolve, reject) => {
+ helper.log_to_file_queue(uuid, "[Checking] " + helper.get_site_from_url(site.url))
+ let driver = undefined
+ if (helper.grid_url == "") {
+ driver = new Builder()
+ .forBrowser("firefox")
+ .setFirefoxOptions(new firefox.Options().headless().windowSize({
+ width: 640,
+ height: 480
+ }))
+ .build();
+ } else {
+ driver = new Builder()
+ .forBrowser("firefox")
+ .setFirefoxOptions(new firefox.Options().headless().windowSize({
+ width: 640,
+ height: 480
+ }))
+ .usingServer(helper.grid_url)
+ .build();
+ }
+
+ try {
+
+ var timeouts = {
+ implicit: 0,
+ pageLoad: 5000,
+ script: 5000
+ };
+
+ var timeout = (site.timeout != 0) ? site.timeout * 1000 : 5000;
+ var implicit = (site.implicit != 0) ? site.implicit * 1000 : 0;
+
+ timeouts = {
+ implicit: implicit,
+ pageLoad: timeout,
+ script: timeout
+ };
+
+ helper.verbose && console.log(timeouts)
+
+ var source = "";
+ var data = "";
+ var text_only = "unavailable";
+ var title = "unavailable";
+ var temp_profile = {
+ "found": 0,
+ "image": "",
+ "link": "",
+ "rate": "",
+ "title": "",
+ "text": "",
+ "type": ""
+ };
+ var link = site.url.replace("{username}", username);
+ await driver.manage().setTimeouts(timeouts);
+ await driver.get(link);;
+ source = await driver.getPageSource();
+ data = await driver.takeScreenshot();
+ title = await driver.getTitle();
+ text_only = await driver.findElement(By.tagName("body")).getText();
+ await driver.quit()
+ if (options.includes("ShowUserProfilesSlow")) {
+ temp_profile["image"] = "data:image/png;base64,{image}".replace("{image}", data);
+ }
+ if (site.selected == "true" && site.detections.length > 0 && options.includes("FindUserProfilesSlow")) {
+ await Promise.all(site.detections.map(async detection => {
+ try {
+ if ("status" in detection) {
+ if (detection.status == "bad") {
+ return;
+ }
+ }
+ var temp_found = "false"
+ if (detection.type == "ocr" && data != "") {
+ tmpobj = tmp.fileSync();
+ fs.writeFileSync(tmpobj.name, Buffer.from(data, "base64"));
+ await tesseract.recognize(tmpobj.name, {
+ lang: "eng",
+ oem: 1,
+ psm: 3,
+ })
+ .then(text => {
+ text = text.replace(/[^A-Za-z0-9]/gi, "");
+ detection.string = detection.string.replace(/[^A-Za-z0-9]/gi, "");
+ if (text != "") {
+ if (text.toLowerCase().includes(detection.string.toLowerCase())) {
+ temp_found = "true";
+ }
+ if (detection.return == temp_found) {
+ //console.log(text);
+ //console.log(detection.string," > Found ocr");
+ temp_profile.found += 1;
+ }
+ }
+ })
+ .catch(error => {
+ helper.verbose && console.log(error.message);
+ })
+ tmpobj.removeCallback();
+ } else if (detection.type == "normal" && source != "") {
+ if (source.toLowerCase().includes(detection.string.replace("{username}", username).toLowerCase())) {
+ temp_found = "true";
+ }
+ if (detection.return == temp_found) {
+ //console.log(detection.string," > normal");
+ temp_profile.found += 1
+ }
+ } else if (detection.type == "advanced" && text_only != "") {
+ if (text_only.toLowerCase().includes(detection.string.replace("{username}", username).toLowerCase())) {
+ temp_found = "true";
+ }
+ if (detection.return == temp_found) {
+ //console.log(detection.string," > normal");
+ temp_profile.found += 1
+ }
+ }
+
+ } catch (err) {
+
+ }
+ }));
+ }
+ if (temp_profile.found > 0 || temp_profile.image != "") {
+ temp_profile.text = sanitizeHtml(text_only);
+ temp_profile.title = sanitizeHtml(title);
+ temp_profile.rate = "%" + ((temp_profile.found / site.detections.length) * 100).toFixed(2);
+ temp_profile.link = site.url.replace("{username}", username);
+ temp_profile.type = site.type
+ resolve(temp_profile);
+ } else {
+ resolve(undefined)
+ }
+ } catch (err) {
+ if (driver !== undefined) {
+ try {
+ await driver.quit()
+ } catch (err) {
+ helper.verbose && console.log("Driver Session Issue")
+ }
+ }
+ resolve(undefined)
+ }
+ });
+}
+
+module.exports = {
+ find_username_advanced
+}
diff --git a/modules/special.js b/modules/special.js
new file mode 100644
index 0000000..fbbfbba
--- /dev/null
+++ b/modules/special.js
@@ -0,0 +1,241 @@
+var helper = require("./helper.js")
+var async = require("async");
+var sanitizeHtml = require("sanitize-html");
+var tesseract = require("node-tesseract-ocr");
+var sanitizeHtml = require("sanitize-html");
+var firefox = require("selenium-webdriver/firefox");
+var {
+ Builder,
+ By,
+ Key
+} = require("selenium-webdriver");
+
+async function find_username_special(req) {
+ const time = new Date();
+ const functions = [];
+ helper.parsed_sites.forEach((site) => {
+ if ("status" in site) {
+ if (site.status == "bad") {
+ return Promise.resolve();
+ }
+ }
+ if (site.selected == "true") {
+ site.detections.forEach((detection) => {
+ if (detection.type == 'special') {
+ if (detection.function == 'special_facebook_1') {
+ functions.push(find_username_site_special_facebook_1.bind(null, req.body.uuid, req.body.string, site));
+ } else if (detection.function == 'special_gmail_1') {
+ functions.push(find_username_site_special_gmail_1.bind(null, req.body.uuid, req.body.string, site));
+ } else if (detection.function == 'special_google_1') {
+ functions.push(find_username_site_special_google_1.bind(null, req.body.uuid, req.body.string, site));
+ }
+ }
+ });
+ }
+ });
+ const results = await async.parallelLimit(functions, 5);
+ helper.verbose && console.log(`Total time ${new Date() - time}`);
+ return results.filter(item => item !== undefined)
+}
+
+async function find_username_site_special_facebook_1(uuid, username, site) {
+ return new Promise(async (resolve, reject) => {
+ helper.log_to_file_queue(uuid, "[Checking] " + helper.get_site_from_url(site.url))
+ let driver = new Builder()
+ .forBrowser("firefox")
+ .setFirefoxOptions(new firefox.Options().headless().windowSize({
+ width: 640,
+ height: 480
+ }))
+ .build();
+
+ try {
+ var timeouts = {
+ implicit: 0,
+ pageLoad: 10000,
+ script: 10000
+ };
+
+ var source = "";
+ var data = "";
+ var text_only = "unavailable";
+ var title = "unavailable";
+ var temp_profile = {
+ "found": 0,
+ "image": "",
+ "link": "",
+ "rate": "",
+ "title": "",
+ "text": "",
+ "type": ""
+ };
+ var link = "https://mbasic.facebook.com/login/identify/?ctx=recoveqr";
+ await driver.manage().setTimeouts(timeouts);
+ await driver.get(link);;
+ await driver.findElement(By.id('identify_search_text_input')).sendKeys(username);
+ await driver.findElement(By.id('did_submit')).click();
+ source = await driver.getPageSource();
+ text_only = await driver.findElement(By.tagName("body")).getText();
+ await driver.quit()
+ if (source.includes("Try Entering Your Password")) {
+ temp_found = "true";
+ temp_profile.found += 1
+ }
+ if (temp_profile.found > 0) {
+ temp_profile.text = "unavailable";
+ temp_profile.title = "unavailable";
+ temp_profile.rate = "%" + ((temp_profile.found / 1) * 100).toFixed(2);
+ temp_profile.link = site.url.replace("{username}", username);
+ temp_profile.type = site.type
+ resolve(temp_profile);
+ } else {
+ resolve(undefined)
+ }
+ } catch (err) {
+ if (driver !== undefined) {
+ try {
+ await driver.quit()
+ } catch (err) {
+ helper.verbose && console.log("Driver Session Issue")
+ }
+ }
+ resolve(undefined)
+ }
+ });
+}
+
+async function find_username_site_special_gmail_1(uuid, username, site) {
+ return new Promise(async (resolve, reject) => {
+ helper.log_to_file_queue(uuid, "[Checking] " + helper.get_site_from_url(site.url))
+ let driver = new Builder()
+ .forBrowser("firefox")
+ .setFirefoxOptions(new firefox.Options().headless().windowSize({
+ width: 640,
+ height: 480
+ }))
+ .build();
+
+ try {
+ var timeouts = {
+ implicit: 0,
+ pageLoad: 10000,
+ script: 10000
+ };
+
+ var source = "";
+ var data = "";
+ var text_only = "unavailable";
+ var title = "unavailable";
+ var temp_profile = {
+ "found": 0,
+ "image": "",
+ "link": "",
+ "rate": "",
+ "title": "",
+ "text": "",
+ "type": ""
+ };
+ var link = "https://accounts.google.com/signup/v2/webcreateaccount?service=mail&continue=https%3A%2F%2Fmail.google.com%2Fmail%2F%3Fpc%3Dtopnav-about-n-en&flowName=GlifWebSignIn&flowEntry=SignUp";
+ await driver.manage().setTimeouts(timeouts);
+ await driver.get(link);;
+ await driver.findElement(By.id('username')).sendKeys(username);
+ await driver.findElement(By.id('selectioni1')).click();
+ source = await driver.getPageSource();
+ text_only = await driver.findElement(By.tagName("body")).getText();
+ await driver.quit()
+ if (text_only.includes("That username is taken") && !text_only.includes('your username must be between') && !text_only.includes('You can use letters')) {
+ temp_found = "true";
+ temp_profile.found += 1
+ }
+ if (temp_profile.found > 0) {
+ temp_profile.text = username + "@gmail.com";
+ temp_profile.title = "unavailable";
+ temp_profile.rate = "%" + ((temp_profile.found / 1) * 100).toFixed(2);
+ temp_profile.link = "https://google.com"
+ temp_profile.type = site.type
+ resolve(temp_profile);
+ } else {
+ resolve(undefined)
+ }
+ } catch (err) {
+ if (driver !== undefined) {
+ try {
+ await driver.quit()
+ } catch (err) {
+ helper.verbose && console.log("Driver Session Issue")
+ }
+ }
+ resolve(undefined)
+ }
+ });
+}
+
+async function find_username_site_special_google_1(uuid, username, site) {
+ return new Promise(async (resolve, reject) => {
+ helper.log_to_file_queue(uuid, "[Checking] " + helper.get_site_from_url(site.url))
+ let driver = new Builder()
+ .forBrowser("firefox")
+ .setFirefoxOptions(new firefox.Options().headless().windowSize({
+ width: 640,
+ height: 480
+ }))
+ .build();
+
+ try {
+ var timeouts = {
+ implicit: 0,
+ pageLoad: 10000,
+ script: 10000
+ };
+
+ var source = "";
+ var data = "";
+ var text_only = "unavailable";
+ var title = "unavailable";
+ var temp_profile = {
+ "found": 0,
+ "image": "",
+ "link": "",
+ "rate": "",
+ "title": "",
+ "text": "",
+ "type": ""
+ };
+ var link = "https://accounts.google.com/signin/v2/identifier?continue=https%3A%2F%2Fmail.google.com%2Fmail%2F&service=mail&sacu=1&rip=1&flowName=GlifWebSignIn&flowEntry=ServiceLogin";
+ await driver.manage().setTimeouts(timeouts);
+ await driver.get(link);;
+ await driver.findElement(By.id('identifierId')).sendKeys(username);
+ await driver.findElement(By.xpath("//button[contains(.,'Next')]")).click();
+ source = await driver.getPageSource();
+ text_only = await driver.findElement(By.tagName("body")).getText();
+ await driver.quit()
+ if (text_only.includes("Couldn't sign you in") && !text_only.includes("Couldn't find your")) {
+ temp_found = "true";
+ temp_profile.found += 1
+ }
+ if (temp_profile.found > 0) {
+ temp_profile.text = username + "@gmail.com";
+ temp_profile.title = "unavailable";
+ temp_profile.rate = "%" + ((temp_profile.found / 1) * 100).toFixed(2);
+ temp_profile.link = "https://google.com"
+ temp_profile.type = site.type
+ resolve(temp_profile);
+ } else {
+ resolve(undefined)
+ }
+ } catch (err) {
+ if (driver !== undefined) {
+ try {
+ await driver.quit()
+ } catch (err) {
+ helper.verbose && console.log("Driver Session Issue")
+ }
+ }
+ resolve(undefined)
+ }
+ });
+}
+
+module.exports = {
+ find_username_special
+}
diff --git a/modules/stringanalysis.js b/modules/stringanalysis.js
new file mode 100644
index 0000000..17904ca
--- /dev/null
+++ b/modules/stringanalysis.js
@@ -0,0 +1,229 @@
+var helper = require("./helper.js")
+var {
+ findWord
+} = require("most-common-words-by-language");
+var fs = require("fs");
+var parsed_json = JSON.parse(fs.readFileSync("dict.json"));
+var WordsNinjaPack = require("wordsninja");
+var WordsNinja = new WordsNinjaPack();
+
+function most_common1(all_words, temp_words) {
+ var temp_list = []
+ Object.keys(all_words).forEach(function(key) {
+ all_words[key].forEach(function(item) {
+ if (!temp_list.includes(item) && item.length > 1) {
+ temp_list.push(item);
+ var temp = findWord(item);
+ if (Object.keys(temp).length != 0) {
+ var languages = Object.keys(temp).map(function(key) {
+ return [key, temp[key]];
+ });
+ languages.sort(function(first, second) {
+ return second[1] - first[1]
+ }).reverse();
+ temp_words.push({
+ "word": item,
+ "languages": languages.map(e => e.join(":")).join(" ")
+ });
+ }
+ }
+ });
+ });
+}
+
+async function most_common(all_words, temp_words) {
+ var temp_list = []
+ Object.keys(all_words).forEach(function(key) {
+ all_words[key].forEach(function(item) {
+ if (!temp_list.includes(item) && item.length > 1) {
+ temp_list.push(item);
+ var temp = findWord(item);
+ if (Object.keys(temp).length != 0) {
+ var languages = Object.keys(temp).map(function(key) {
+ return [key, temp[key]];
+ });
+ languages.sort(function(first, second) {
+ return second[1] - first[1]
+ }).reverse();
+ temp_words.push({
+ "word": item,
+ "languages": languages.map(e => e[0]).join(", ")
+ });
+ }
+ }
+ });
+ });
+}
+
+function find_other(all_words) {
+ var words = WordsNinja.splitSentence(req.body.string);
+
+ words.forEach(function(word) {
+ var value = false
+ Object.keys(all_words).forEach(function(key) {
+ if (all_words[key].includes(word)) {
+ value = true
+ }
+ });
+
+ if (!value && !all_words.maybe.includes(word)) {
+ all_words.maybe.push(word);
+ }
+ });
+}
+
+function remove_word(str, sub_string) {
+ part1 = str.substring(0, str.indexOf(sub_string));
+ part2 = str.substring(str.indexOf(sub_string) + sub_string.length, str.length);
+ temp = (part1 + part2).replace(/[ \[\]:"\\|,.<>\/?~`!@#$%^&*()_+\-={};"]/gi, "");
+ return temp;
+}
+
+async function analyze_string(req, all_words) {
+ helper.log_to_file_queue(req.body.uuid, "[Starting] String analysis")
+ temp_rr_names = []
+ string_to_check = req.body.string
+ parsed_json.prefix.forEach(function(item, index) {
+ if (string_to_check.indexOf(item) == 0 && !all_words.prefix.includes(item)) {
+ all_words.prefix.push(item);
+ temp = remove_word(string_to_check, item);
+ if (temp !== null && temp !== "" && !all_words.unknown.includes(temp) && !all_words.maybe.includes(temp) && temp.length > 1) {
+ all_words.unknown.push(temp);
+ }
+ }
+ });
+ parsed_json.m_names.forEach(function(item, index) {
+ if (string_to_check.indexOf(item) >= 0 && !all_words.name.includes(item)) {
+ all_words.name.push(item);
+ temp = remove_word(string_to_check, item);
+ if (temp !== null && temp !== "" && !all_words.unknown.includes(temp) && !all_words.maybe.includes(temp) && temp.length > 1) {
+ all_words.unknown.push(temp);
+ }
+ }
+ });
+ parsed_json.f_names.forEach(function(item, index) {
+ if (string_to_check.indexOf(item) >= 0 && !all_words.name.includes(item)) {
+ all_words.name.push(item);
+ temp = remove_word(string_to_check, item);
+ if (temp !== null && temp !== "" && !all_words.unknown.includes(temp) && !all_words.maybe.includes(temp) && temp.length > 1) {
+ all_words.unknown.push(temp);
+ }
+ }
+ });
+
+ all_words.prefix.forEach(function(h_item, index) {
+ all_words.unknown.forEach(function(r_item, index) {
+ if (r_item.indexOf(h_item) == 0) {
+ temp = remove_word(r_item, h_item);
+ if (temp !== null && temp !== "" && !temp_rr_names.includes(temp) && !all_words.maybe.includes(temp) && temp.length > 1) {
+ temp_rr_names.push(temp);
+ }
+ }
+ });
+ });
+
+ var temp_r_concat = all_words.unknown.concat(temp_rr_names.filter((item) => all_words.unknown.indexOf(item) < 0));
+
+ all_words.unknown = temp_r_concat
+ temp_rr_names = []
+
+ all_words.number.forEach(function(n_item, index) {
+ all_words.unknown.forEach(function(r_item, index) {
+ if (r_item.indexOf(n_item) >= 0) {
+ temp = remove_word(r_item, n_item);
+ if (temp !== null && temp !== "" && !temp_rr_names.includes(temp) && !all_words.maybe.includes(temp) && temp.length > 1) {
+ temp_rr_names.push(temp);
+ }
+ }
+ });
+ });
+
+ var temp_r_concat = all_words.unknown.concat(temp_rr_names.filter((item) => all_words.unknown.indexOf(item) < 0));
+ all_words.unknown = temp_r_concat
+ helper.log_to_file_queue(req.body.uuid, "[Done] String analysis")
+}
+
+async function split_upper_case(req, all_words) {
+ try {
+ req.body.string.match(/[A-Z][a-z]+/g).forEach((item) => {
+ if (item.length > 1 && !all_words.unknown.includes(item) && !all_words.maybe.includes(item)) {
+ all_words.unknown.push(item.toLowerCase());
+ }
+ });
+ } catch (err) {}
+}
+
+async function split_alphabet_case(req, all_words) {
+ try {
+ req.body.string.match(/[A-Za-z]+/g).forEach((item) => {
+ if (item.length > 1 && !all_words.unknown.includes(item) && !all_words.maybe.includes(item)) {
+ all_words.unknown.push(item.toLowerCase());
+ }
+ });
+ } catch (err) {}
+}
+
+async function find_symbols(req, all_words) {
+ try {
+ req.body.string.match(/[ \[\]:"\\|,.<>\/?~`!@#$%^&*()_+\-={};']/gi).forEach((item) => {
+ if (item !== " " && !all_words.symbol.includes(item)) {
+ all_words.symbol.push(item);
+ }
+ });
+ } catch (err) {}
+}
+
+async function find_numbers(req, all_words) {
+ try {
+ req.body.string.match(/(\d+)/g).forEach((item) => {
+ if (!all_words.number.includes(item)) {
+ all_words.number.push(item);
+ }
+ });
+ } catch (err) {}
+}
+
+async function convert_numbers(req) {
+ try {
+ numbers_to_letters = {
+ "4": "a",
+ "8": "b",
+ "3": "e",
+ "1": "l",
+ "0": "o",
+ "5": "s",
+ "7": "t",
+ "2": "z"
+ }
+
+ temp_value = ""
+ for (i = 0; i < req.body.string.length; i++) {
+ _temp = numbers_to_letters[req.body.string.charAt(i)]
+ if (_temp != undefined) {
+ temp_value += numbers_to_letters[req.body.string.charAt(i)];
+ } else {
+ temp_value += req.body.string.charAt(i);
+ }
+ }
+ req.body.string = temp_value
+ } catch (err) {}
+}
+
+async function get_maybe_words(req, all_words) {
+ await WordsNinja.loadDictionary();
+ all_words.maybe = await WordsNinja.splitSentence(req.body.string).filter(function(elem, index, self) {
+ return index === self.indexOf(elem);
+ }).filter(word => word.length > 1);
+}
+
+module.exports = {
+ get_maybe_words,
+ find_symbols,
+ find_numbers,
+ convert_numbers,
+ split_upper_case,
+ split_alphabet_case,
+ most_common,
+ find_other,
+ analyze_string
+}
diff --git a/package.json b/package.json
index f2bb610..e515bb4 100644
--- a/package.json
+++ b/package.json
@@ -1,14 +1,15 @@
{
"name": "social-analyzer",
- "version": "2.0.2",
+ "version": "2.0.10",
"description": "Advanced Social Media Profiles Finder and String Analysis Tool",
- "main": "server.js",
+ "main": "app.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1",
"start": "node app.js"
},
"dependencies": {
"async": "3.2.0",
+ "cheerio": "1.0.0-rc.5",
"express": "^4.16.3",
"express-async-errors": "3.1.1",
"follow-redirects": "1.13.0",
@@ -28,8 +29,7 @@
"tmp": "0.2.1",
"wink-tokenizer": "1.1.0",
"wordsninja": "1.0.0",
- "yargs": "16.2.0",
- "cheerio": "1.0.0-rc.5"
+ "yargs": "16.2.0"
},
"author": "QeeqBox",
"license": "AGPL-3.0",
diff --git a/public/app.html b/public/app.html
index 297d252..eb3ca68 100644
--- a/public/app.html
+++ b/public/app.html
@@ -1,6 +1,6 @@