diff --git a/README.md b/README.md index 841d977..2d2e01f 100644 --- a/README.md +++ b/README.md @@ -5,18 +5,18 @@ API and Web App for analyzing & finding a person profile across +300 social media websites. It includes different string analysis and detection modules, you can choose which combination of modules to use during the investigation. -The detection modules utilize a rating mechanism based on different detection techniques, which produces a rate value that starts from 0 to 100 (No-Maybe-Yes) +The detection modules utilize a rating mechanism based on different detection techniques, which produces a rate value that starts from 0 to 100 (No-Maybe-Yes). This model produces less *False Positive* unlike other projects This project could help in investigating profiles related to suspicious or malicious activities such as [cyberbullying](https://en.wikipedia.org/wiki/Wikipedia:Cyberbullying), [cybergrooming](https://de.wikipedia.org/wiki/Cyber-Grooming) & [cyberstalking](https://en.wikipedia.org/wiki/Cyberstalking), [spreading misinformation](https://en.wikipedia.org/wiki/Misinformation). And, *"It's currently used by some law enforcement agencies in countries where resources are limited"* **Pull Requests are welcomed!** ## Updates -- (Testing) A new refactored version (Coming soon!!) -- Added special detection for google 👏 -- Added more customization to the CLI python version (It's a lot easier to integrate using the JSON output switch) 👏 -- Added Custom Search Query 👏 -- Added similarity checking to FindOrigins 👏 +- Added a new refactored version 👏👏👏 +- Added special detection for google +- Added more customization to the CLI python version (It's a lot easier to integrate using the JSON output switch) +- Added Custom Search Query +- Added similarity checking to FindOrigins ## Security Testing @@ -52,11 +52,11 @@ Profile images **will not** be blurred. If you want them to be blurred, turn tha - Custom user-agent, proxy, timeout & implicit wait - Python CLI & NodeJS CLI (Limited to FindUserProfilesFast option) - Grid Option for Faster Checking (Limited to docker-compose) 👏 -- Dump Logs to folder or terminal +- Dump Logs to folder or terminal (prettified) - [Wiki](https://github.com/qeeqbox/social-analyzer/wiki) ## Special Detections -- Facebook +- Facebook, @gmail and google ## Install and run as web app (NodeJS + NPM + Firefox) ```bash @@ -81,9 +81,9 @@ cd social-analyzer rm -rf package-lock.json node_modules npm install lodash npm install -# If you want to list all websites use node app.js -c -l +# If you want to list all websites use node app.js --cli --list # Remember the following runs as FindUserProfilesFast -node app.js -c -m "fast" -u "username" -w "youtube pinterest tumblr" +node app.j --cli --mode "fast" --username "johndoe" --websites "youtube pinterest tumblr" --output "pretty" ``` ## Install and run as CLI (Python3 + NPM + Firefox) @@ -94,7 +94,7 @@ sudo apt-get install -y firefox-esr tesseract-ocr git git clone https://github.com/qeeqbox/social-analyzer.git cd social-analyzer pip3 install lxml BeautifulSoup4 tld pygments -# If you want to list all websites use python3 app.py -c -l +# If you want to list all websites use python3 app.py --cli --list # Remember the following runs as FindUserProfilesFast python3 app.py --cli --mode "fast" --username "johndoe" --websites "youtube pinterest tumblr" --output "pretty" ``` diff --git a/app.js b/app.js index 051ecd7..ee841ea 100644 --- a/app.js +++ b/app.js @@ -8,145 +8,74 @@ // contributors list qeeqbox/social-analyzer/graphs/contributors // ------------------------------------------------------------- -var google_api_key = ""; -var google_api_cs = ""; -var grid_url = ""; -var verbose = false -var proxy = "" +var argv = require('yargs') + .usage('Usage: $0 --cli --mode "fast" --username "johndoe" --websites "youtube tiktok"\nUsage: $0 --cli --mode "fast" --username "johndoe"') + .describe('cli', 'a user or stirng') + .default("cli", false) + .boolean('cli') + .describe('username', 'E.g. johndoe, john_doe or johndoe9999') + .default("username", "") + .describe('websites', 'Website or websites separated by space E.g. youtube, tiktok or tumblr') + .default("websites", "all") + .describe('mode', 'Analysis mode E.g.fast -> FindUserProfilesFast, slow -> FindUserProfilesSlow or special -> FindUserProfilesSpecial') + .default("mode", "") + .describe('output', 'arg_parser_optional.add_argument("--output", help="Show the output in the following format: json -> json output for integration or pretty -> prettify the output') + .default("output", "") + .describe('options', 'Show the following when a profile is found: link, rate, title or text') + .default("options", "") + .describe('list', 'List all available websites') + .default("list", false) + .boolean('list') + .describe('grid', 'grid option, not for CLI') + .default("grid", "") + .help('help') + .argv; -var header_options = { - headers: { - 'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:84.0) Gecko/20100101 Firefox/84.0', - } -}; - -console.log('[!] Detections are updated every often, make sure to get the most updated ones'); +if (argv.output != "json") { + console.log('[!] Detections are updated every often, make sure to get the most updated ones'); +} var semver = require('semver'); if (semver.satisfies(process.version, '>13 || <13')) { - console.log('[Good] NodeJS Version Check'); + if (argv.output != "json") { + console.log('[Good] NodeJS Version Check'); + } } else { - console.log('[Error] NodeJS Version Check'); + if (argv.output != "json") { + console.log('[Error] NodeJS Version Check'); + } process.exit(1); } -var argv = require('yargs') - .alias('c', 'cli') - .alias('u', 'user') - .alias('w', 'website') - .alias('o', 'output') - .alias('l', 'list') - .alias('m', 'mode') - .usage('Usage: $0 -c -m "mode" -u "user" -w "website[s]" -o "output"') - .example('$0 -c -m "fast" -u "joe" -w "facebook"') - .example('$0 -c -m "fast" -u "natalie" -w "facebook wordpress"') - .describe('u', 'a user or stirng') - .describe('w', 'a website or websites sparated with space') - .describe('o', 'option output file') - .describe('l', 'list all available websites') - .describe('m', 'fast -> FindUserProfilesFast\nslow -> FindUserProfilesSlow\nspecial -> FindUserProfilesSpecial') - .help('h') - .alias('h', 'help') - .argv; - -var tmp = require("tmp"); var express = require("express"); var fs = require("fs"); var tokenizer = require("wink-tokenizer"); -var WordsNinjaPack = require("wordsninja"); var generatorics = require("generatorics"); -var { - findWord -} = require("most-common-words-by-language"); -var tesseract = require("node-tesseract-ocr"); -var url = require("url"); -var sanitizeHtml = require("sanitize-html"); -var firefox = require("selenium-webdriver/firefox"); -var { - Builder, - By, - Key -} = require("selenium-webdriver"); -var util = require('util'); -var https = require("follow-redirects").https; -var async = require("async"); var HttpsProxyAgent = require('https-proxy-agent'); var PrettyError = require('pretty-error'); -var stringSimilarity = require("string-similarity"); var pe = new PrettyError(); require('express-async-errors'); -//var jsdom = require('jsdom'); -//var dom = new jsdom.JSDOM(); -//var window = dom.window; -//var document = window.document; -//var $ = require('jquery')(window); -const { - htmlToText -} = require('html-to-text'); -var cheerio = require('cheerio'); var _tokenizer = tokenizer(); -var parsed_json = JSON.parse(fs.readFileSync("dict.json")); -var parsed_sites = JSON.parse(fs.readFileSync("sites.json")); -var parsed_names_origins = JSON.parse(fs.readFileSync("names.json")); -var app = express(); - -var WordsNinja = new WordsNinjaPack(); -app.use(express.urlencoded({ - extended: true -})); -app.use(express.json()); -app.use(express.static("public")); +var helper = require("./modules/helper.js") +var fastscan = require("./modules/fastscan.js") +var slowscan = require("./modules/slowscan.js") +var special = require("./modules/special.js") +var externalapis = require("./modules/externalapis.js") +var stringanalysis = require("./modules/stringanalysis.js") +var nameanalysis = require("./modules/nameanalysis.js") if (!fs.existsSync('logs')) { fs.mkdirSync('logs'); } -var logs_queue = Promise.resolve(); +var app = express(); -function log_to_file_queue(uuid, msg) { - logs_queue = logs_queue.then(function() { - return new Promise(function(resolve) { - fs.appendFile("logs/" + uuid + "_log.txt", msg + "\n", function(err, data) { - console.log(msg) - resolve(); - }); - }); - }); -} - -async function get_url_wrapper_json(url, time = 5) { - try { - let http_promise = new Promise((resolve, reject) => { - var request = https.get(url, header_options, function(res) { - var body = "" - res.on("data", function(chunk) { - body += chunk; - }); - res.on("end", function() { - resolve({ - 'data': JSON.parse(body.toString()) - }); - }); - }); - request.on('error', function(e) { - reject({ - 'data': '' - }) - }); - request.on('socket', function(socket) { - var timeout = (time != 0) ? time * 1000 : 5000; - socket.setTimeout(timeout, function() { - request.abort(); - }); - }); - }); - let response_body = await http_promise; - return response_body - } catch (err) { - verbose && console.log(err); - } -} +app.use(express.urlencoded({ + extended: true +})); +app.use(express.json()); +app.use(express.static("public")); app.post("/get_logs", async function(req, res, next) { var last_line = "nothinghere" @@ -160,610 +89,9 @@ app.post("/get_logs", async function(req, res, next) { } }) -function get_site_from_url(_url) { - temp = url.parse(_url.replace("{username}", "nothinghere")).hostname - return temp.replace("nothinghere.", "") -} - -async function find_username_special(req) { - const time = new Date(); - const functions = []; - parsed_sites.forEach((site) => { - if ("status" in site) { - if (site.status == "bad") { - return Promise.resolve(); - } - } - if (site.selected == "true") { - site.detections.forEach((detection) => { - if (detection.type == 'special') { - if (detection.function == 'special_facebook_1') { - functions.push(find_username_site_special_facebook_1.bind(null, req.body.uuid, req.body.string, site)); - } else if (detection.function == 'special_google_1') { - functions.push(find_username_site_special_google_1.bind(null, req.body.uuid, req.body.string, site)); - } - } - }); - } - }); - const results = await async.parallelLimit(functions, 5); - verbose && console.log(`Total time ${new Date() - time}`); - return results.filter(item => item !== undefined) -} - -async function find_username_site_special_facebook_1(uuid, username, site) { - return new Promise(async (resolve, reject) => { - log_to_file_queue(uuid, "[Checking] " + get_site_from_url(site.url)) - let driver = new Builder() - .forBrowser("firefox") - .setFirefoxOptions(new firefox.Options().headless().windowSize({ - width: 640, - height: 480 - })) - .build(); - - try { - var timeouts = { - implicit: 0, - pageLoad: 10000, - script: 10000 - }; - - var source = ""; - var data = ""; - var text_only = "unavailable"; - var title = "unavailable"; - var temp_profile = { - "found": 0, - "image": "", - "link": "", - "rate": "", - "title": "", - "text": "", - "type": "" - }; - var link = "https://mbasic.facebook.com/login/identify/?ctx=recoveqr"; - await driver.manage().setTimeouts(timeouts); - await driver.get(link);; - await driver.findElement(By.id('identify_search_text_input')).sendKeys(username); - await driver.findElement(By.id('did_submit')).click(); - source = await driver.getPageSource(); - text_only = await driver.findElement(By.tagName("body")).getText(); - await driver.quit() - if (source.includes("Try Entering Your Password")) { - temp_found = "true"; - temp_profile.found += 1 - } - if (temp_profile.found > 0) { - temp_profile.text = "unavailable"; - temp_profile.title = "unavailable"; - temp_profile.rate = "%" + ((temp_profile.found / 1) * 100).toFixed(2); - temp_profile.link = site.url.replace("{username}", username); - temp_profile.type = site.type - resolve(temp_profile); - } else { - resolve(undefined) - } - } catch (err) { - if (driver !== undefined) { - try { - await driver.quit() - } catch (err) { - verbose && console.log("Driver Session Issue") - } - } - resolve(undefined) - } - }); -} - -async function find_username_site_special_google_1(uuid, username, site) { - return new Promise(async (resolve, reject) => { - log_to_file_queue(uuid, "[Checking] " + get_site_from_url(site.url)) - let driver = new Builder() - .forBrowser("firefox") - .setFirefoxOptions(new firefox.Options().headless().windowSize({ - width: 640, - height: 480 - })) - .build(); - - try { - var timeouts = { - implicit: 0, - pageLoad: 10000, - script: 10000 - }; - - var source = ""; - var data = ""; - var text_only = "unavailable"; - var title = "unavailable"; - var temp_profile = { - "found": 0, - "image": "", - "link": "", - "rate": "", - "title": "", - "text": "", - "type": "" - }; - var link = "https://accounts.google.com/signup/v2/webcreateaccount?service=mail&continue=https%3A%2F%2Fmail.google.com%2Fmail%2F%3Fpc%3Dtopnav-about-n-en&flowName=GlifWebSignIn&flowEntry=SignUp"; - await driver.manage().setTimeouts(timeouts); - await driver.get(link);; - await driver.findElement(By.id('username')).sendKeys(username); - await driver.findElement(By.id('selectioni1')).click(); - source = await driver.getPageSource(); - text_only = await driver.findElement(By.tagName("body")).getText(); - await driver.quit() - if (text_only.includes("That username is taken") && !text_only.includes('your username must be between') && !text_only.includes('You can use letters')) { - temp_found = "true"; - temp_profile.found += 1 - } - if (temp_profile.found > 0) { - temp_profile.text = username + "@gmail.com"; - temp_profile.title = "unavailable"; - temp_profile.rate = "%" + ((temp_profile.found / 1) * 100).toFixed(2); - temp_profile.link = "https://google.com" - temp_profile.type = site.type - resolve(temp_profile); - } else { - resolve(undefined) - } - } catch (err) { - if (driver !== undefined) { - try { - await driver.quit() - } catch (err) { - verbose && console.log("Driver Session Issue") - } - } - resolve(undefined) - } - }); -} - -async function find_username_advanced(req) { - const time = new Date(); - const functions = []; - parsed_sites.forEach((site) => { - if ("status" in site) { - if (site.status == "bad") { - return Promise.resolve(); - } - } - if (site.selected == "true" && site.detections.length > 0) { - functions.push(find_username_site_new.bind(null, req.body.uuid, req.body.string, req.body.option, site)); - } - }); - const results = await async.parallelLimit(functions, 8); - verbose && console.log(`Total time ${new Date() - time}`); - return results.filter(item => item !== undefined) -} - -async function find_username_site_new(uuid, username, options, site) { - return new Promise(async (resolve, reject) => { - log_to_file_queue(uuid, "[Checking] " + get_site_from_url(site.url)) - let driver = undefined - if (grid_url == "") { - driver = new Builder() - .forBrowser("firefox") - .setFirefoxOptions(new firefox.Options().headless().windowSize({ - width: 640, - height: 480 - })) - .build(); - } else { - driver = new Builder() - .forBrowser("firefox") - .setFirefoxOptions(new firefox.Options().headless().windowSize({ - width: 640, - height: 480 - })) - .usingServer(grid_url) - .build(); - } - - - try { - - var timeouts = { - implicit: 0, - pageLoad: 5000, - script: 5000 - }; - - var timeout = (site.timeout != 0) ? site.timeout * 1000 : 5000; - var implicit = (site.implicit != 0) ? site.implicit * 1000 : 0; - - timeouts = { - implicit: implicit, - pageLoad: timeout, - script: timeout - }; - - verbose && console.log(timeouts) - - var source = ""; - var data = ""; - var text_only = "unavailable"; - var title = "unavailable"; - var temp_profile = { - "found": 0, - "image": "", - "link": "", - "rate": "", - "title": "", - "text": "", - "type": "" - }; - var link = site.url.replace("{username}", username); - await driver.manage().setTimeouts(timeouts); - await driver.get(link);; - source = await driver.getPageSource(); - data = await driver.takeScreenshot(); - title = await driver.getTitle(); - text_only = await driver.findElement(By.tagName("body")).getText(); - await driver.quit() - if (options.includes("ShowUserProfilesSlow")) { - temp_profile["image"] = "data:image/png;base64,{image}".replace("{image}", data); - } - if (site.selected == "true" && site.detections.length > 0 && options.includes("FindUserProfilesSlow")) { - await Promise.all(site.detections.map(async detection => { - try { - if ("status" in detection) { - if (detection.status == "bad") { - return; - } - } - var temp_found = "false" - if (detection.type == "ocr" && data != "") { - tmpobj = tmp.fileSync(); - fs.writeFileSync(tmpobj.name, Buffer.from(data, "base64")); - await tesseract.recognize(tmpobj.name, { - lang: "eng", - oem: 1, - psm: 3, - }) - .then(text => { - text = text.replace(/[^A-Za-z0-9]/gi, ""); - detection.string = detection.string.replace(/[^A-Za-z0-9]/gi, ""); - if (text != "") { - if (text.toLowerCase().includes(detection.string.toLowerCase())) { - temp_found = "true"; - } - if (detection.return == temp_found) { - //console.log(text); - //console.log(detection.string," > Found ocr"); - temp_profile.found += 1; - } - } - }) - .catch(error => { - verbose && console.log(error.message); - }) - tmpobj.removeCallback(); - } else if (detection.type == "normal" && source != "") { - if (source.toLowerCase().includes(detection.string.replace("{username}", username).toLowerCase())) { - temp_found = "true"; - } - if (detection.return == temp_found) { - //console.log(detection.string," > normal"); - temp_profile.found += 1 - } - } else if (detection.type == "advanced" && text_only != "") { - if (text_only.toLowerCase().includes(detection.string.replace("{username}", username).toLowerCase())) { - temp_found = "true"; - } - if (detection.return == temp_found) { - //console.log(detection.string," > normal"); - temp_profile.found += 1 - } - } - - } catch (err) { - - } - })); - } - if (temp_profile.found > 0 || temp_profile.image != "") { - temp_profile.text = sanitizeHtml(text_only); - temp_profile.title = sanitizeHtml(title); - temp_profile.rate = "%" + ((temp_profile.found / site.detections.length) * 100).toFixed(2); - temp_profile.link = site.url.replace("{username}", username); - temp_profile.type = site.type - resolve(temp_profile); - } else { - resolve(undefined) - } - } catch (err) { - if (driver !== undefined) { - try { - await driver.quit() - } catch (err) { - verbose && console.log("Driver Session Issue") - } - } - resolve(undefined) - } - }); -} - -async function find_username_normal(req) { - - var functions = []; - var detections_result = []; - - async function find_username_site(uuid, username, options, site, body) { - try { - log_to_file_queue(uuid, "[Checking] " + get_site_from_url(site.url)) - var detections_count = 0; - var source = body; - var text_only = "unavailable"; - var title = "unavailable"; - var temp_profile = { - "found": 0, - "image": "", - "link": "", - "rate": "", - "title": "", - "text": "", - "type": "" - }; - - await Promise.all(site.detections.map(async detection => { - var temp_found = "false"; - if (detection.type == "normal" && options.includes("FindUserProfilesFast") && source != "") { - detections_count += 1 - if (source.toLowerCase().includes(detection.string.replace("{username}", username).toLowerCase())) { - temp_found = "true"; - } - if (detection.return == temp_found) { - //console.log(detection.string, " > normal"); - temp_profile.found += 1 - } - } - })); - if (temp_profile.found > 0 && detections_count != 0) { - temp_profile.text = sanitizeHtml(htmlToText(body, { - wordwrap: false, - hideLinkHrefIfSameAsText: true, - ignoreHref: true, - ignoreImage: true - })); - if (temp_profile.text == "") { - temp_profile.text = "unavailable" - } - - try { - var $ = cheerio.load(body); - title = sanitizeHtml($("title").text()) - if (title.length == 0) { - title = "unavailable" - } - } catch (err) { - verbose && console.log(err); - } - - temp_profile.title = title; - temp_profile.rate = "%" + ((temp_profile["found"] / detections_count) * 100).toFixed(2); - temp_profile.link = site.url.replace("{username}", username); - temp_profile.type = site.type - return Promise.resolve(temp_profile); - } - return Promise.resolve(); - } catch (err) { - return Promise.reject(); - } - } - - async function find_username_sites(uuid, username, options, parsed_sites) { - - await parsed_sites.forEach(site => { - if ("status" in site) { - if (site.status == "bad") { - return; - } - } - if (site.selected == "true" && site.detections.length > 0) { - functions.push(function(callback) { - var request = https.get(site.url.replace("{username}", username), header_options, function(res) { - var body = "" - res.on("data", function(chunk) { - body += chunk; - }); - res.on("end", async function() { - var results = await find_username_site(uuid, username, options, site, body); - detections_result.push(results); - callback(null, "Done!"); - }); - }); - request.on('error', function(e) { - callback(null, "Done!"); - }); - request.on('socket', function(socket) { - var timeout = (site.timeout != 0) ? site.timeout * 1000 : 5000; - socket.setTimeout(timeout, function() { - request.abort(); - }); - }); - }); - } - }); - } - - await find_username_sites(req.body.uuid, req.body.string, req.body.option, parsed_sites); - await async.parallelLimit(functions, 100); - return detections_result.filter(item => item !== undefined); -} - -async function find_username_advanced_2(username, options) { - - var detections_result = []; - - async function find_username_site(username, options, driver, site) { - try { - if ("status" in site) { - if (site.status == "bad") { - return Promise.resolve(); - } - } - if (site.selected == "true" && site.detections.length > 0 || site.selected == "true" && options.includes("ShowUserProfilesSlow")) { - var source = ""; - var data = ""; - var text_only = "unavailable"; - var title = "unavailable"; - var temp_profile = { - "found": 0, - "image": "", - "link": "", - rate: "", - title: "", - text: "" - }; - var link = site.url.replace("{username}", username); - await driver.get(link);; - source = await driver.getPageSource(); - data = await driver.takeScreenshot(); - title = await driver.getTitle(); - text_only = await driver.findElement(By.tagName("body")).getText(); - if (options.includes("ShowUserProfilesSlow")) { - temp_profile["image"] = "data:image/png;base64,{image}".replace("{image}", data); - } - if (site.selected == "true" && site.detections.length > 0 && options.includes("FindUserProfilesSlow")) { - await Promise.all(site.detections.map(async detection => { - if ("status" in detection) { - if (detection.status == "bad") { - return; - } - } - var temp_found = "false" - if (detection.type == "ocr" && data != "") { - tmpobj = tmp.fileSync(); - fs.writeFileSync(tmpobj.name, Buffer.from(data, "base64")); - await tesseract.recognize(tmpobj.name, { - lang: "eng", - oem: 1, - psm: 3, - }) - .then(text => { - text = text.replace(/[^A-Za-z0-9]/gi, ""); - detection.string = detection.string.replace(/[^A-Za-z0-9]/gi, ""); - if (text != "") { - if (text.toLowerCase().includes(detection.string.toLowerCase())) { - temp_found = "true"; - } - if (detection.return == temp_found) { - //console.log(text); - //console.log(detection.string," > Found ocr"); - temp_profile.found += 1; - } - } - }) - .catch(error => { - verbose && console.log(error.message); - }) - tmpobj.removeCallback(); - } else if (detection.type == "normal" && source != "") { - if (source.toLowerCase().includes(detection.string.replace("{username}", username).toLowerCase())) { - temp_found = "true"; - } - if (detection.return == temp_found) { - //console.log(detection.string," > normal"); - temp_profile.found += 1 - } - } - })); - } - if (temp_profile.found > 0 || temp_profile.image != "") { - temp_profile.text = sanitizeHtml(text_only); - temp_profile.title = sanitizeHtml(title); - temp_profile.rate = "%" + ((temp_profile.found / site.detections.length) * 100).toFixed(2); - temp_profile.link = site.url.replace("{username}", username); - return Promise.resolve(temp_profile); - } - } - return Promise.resolve(); - } catch (err) { - return Promise.reject(); - } - } - - async function find_username_sites(username, options, driver, parsed_sites) { - for (var site of parsed_sites) { - var result = await find_username_site(username, options, driver, site); - detections_result.push(result); - } - - return detections_result; - } - - let driver = new Builder() - .forBrowser("firefox") - .setFirefoxOptions(new firefox.Options().headless().windowSize({ - width: 640, - height: 480 - })) - .build(); - - var timeouts = { - implicit: 0, - pageLoad: 10000, - script: 10000 - }; - - await driver.manage().setTimeouts(timeouts); - var results = await find_username_sites(username, options, driver, parsed_sites); - await driver.quit(); - return results.filter(item => item !== undefined); -} - -async function custom_search_ouputs(req) { - var possible_parameters = ['user', 'profile', 'account'] - const time = new Date(); - const functions = []; - possible_parameters.forEach((key) => { - functions.push(custom_search_ouputs_website.bind(null, req.body.uuid, req.body.string, key)); - }); - const results = await async.parallelLimit(functions, 6); - verbose && console.log(`Total time ${new Date() - time}`); - var merged = [].concat.apply([], results.filter(item => item !== undefined)); - return merged -} - -async function custom_search_ouputs_website(uuid, name, key) { - - return new Promise(async (resolve, reject) => { - try { - var results = [] - log_to_file_queue(uuid, "[Custom Search Using] " + key) - var url = "https://www.googleapis.com/customsearch/v1?key={0}&cx={1}&q={2}:{3}".replace("{0}", google_api_key).replace("{1}", google_api_cs).replace("{2}", key).replace("{3}", name); - var response = await get_url_wrapper_json(url); - if (response.data != '') { - if ('items' in response.data) { - response.data.items.forEach((key) => { - results.push({ - site: get_site_from_url(key.link), - link: key.link, - snippet: key.snippet - }) - }); - } - } - if (results.length > 0) { - resolve(results); - } else { - resolve(undefined) - } - } catch (err) { - resolve(undefined) - } - }); -} - app.get("/get_settings", async function(req, res, next) { temp_list = []; - temp_list = await Promise.all(parsed_sites.map(async (site, index) => { + temp_list = await Promise.all(helper.parsed_sites.map(async (site, index) => { var temp_url = ""; if ("status" in site) { if (site.status == "bad") { @@ -771,8 +99,7 @@ app.get("/get_settings", async function(req, res, next) { } } if (site.detections.length > 0) { - temp_url = url.parse(site.url.replace("{username}", "nothinghere")).hostname - temp_url = temp_url.replace("nothinghere.", ""); + temp_url = helper.get_site_from_url(site.url) if (temp_url != "nothinghere") { temp_selected = "false"; if ("selected" in site) { @@ -801,42 +128,42 @@ app.get("/get_settings", async function(req, res, next) { return 0; }); res.json({ - proxy: proxy, - user_agent: header_options['headers']['User-Agent'], - google: [google_api_key.substring(0, 10) + "******", google_api_cs.substring(0, 10) + "******"], + proxy: helper.proxy, + user_agent: helper.header_options['headers']['User-Agent'], + google: [helper.google_api_key.substring(0, 10) + "******", helper.google_api_cs.substring(0, 10) + "******"], websites: temp_list }); }); app.post("/save_settings", async function(req, res, next) { - await parsed_sites.forEach(function(value, i) { - parsed_sites[i].selected = "false" + await helper.parsed_sites.forEach(function(value, i) { + helper.parsed_sites[i].selected = "false" }); if ("websites" in req.body) { if (req.body.websites.length > 0) { await req.body.websites.split(',').forEach(item => { - parsed_sites[Number(item)].selected = "true"; + helper.parsed_sites[Number(item)].selected = "true"; }); } } - if (req.body.google_key != google_api_key.substring(0, 10) + "******") { - google_api_key = req.body.google_key; + if (req.body.google_key != helper.google_api_key.substring(0, 10) + "******") { + helper.google_api_key = req.body.google_key; } - if (req.body.google_cv != google_api_cs.substring(0, 10) + "******") { - google_api_cs = req.body.google_cv; + if (req.body.google_cv != helper.google_api_cs.substring(0, 10) + "******") { + helper.google_api_cs = req.body.google_cv; } - if (req.body.user_agent != header_options['headers']['User-Agent']) { - header_options['headers']['User-Agent'] = req.body.user_agent; + if (req.body.user_agent != helper.header_options['headers']['User-Agent']) { + helper.header_options['headers']['User-Agent'] = req.body.user_agent; } - if (req.body.proxy != proxy) { - proxy = req.body.proxy; + if (req.body.proxy != helper.proxy) { + helper.proxy = req.body.proxy; } - if (proxy != "") { - header_options['agent'] = HttpsProxyAgent(proxy) + if (helper.proxy != "") { + helper.header_options['agent'] = HttpsProxyAgent(helper.proxy) } else { - if ('agent' in header_options) { - delete header_options['agent']; + if ('agent' in helper.header_options) { + delete helper.header_options['agent']; } } @@ -857,323 +184,10 @@ app.get("/generate", async function(req, res, next) { res.json({ combinations: list_of_combinations }); - }); -async function find_origins(req) { - var found = [] - for (key in parsed_names_origins) { - for (name in parsed_names_origins[key]['boy']) { - if (req.body.string.includes(parsed_names_origins[key]['boy'][name])) { - found.push({ - "name": parsed_names_origins[key]['boy'][name], - "origin": key, - "gender": "boy", - "matched": parsed_names_origins[key]['boy'][name], - "similar": "" - }) - } else { - var similarity = stringSimilarity.compareTwoStrings(req.body.string, parsed_names_origins[key]['boy'][name]); - if (similarity > 0.7) { - found.push({ - "name": req.body.string, - "origin": key, - "gender": "boy", - "matched": "", - "similar": parsed_names_origins[key]['boy'][name] - }) - } - } - } - for (name in parsed_names_origins[key]['girl']) { - if (req.body.string.includes(parsed_names_origins[key]['girl'][name])) { - found.push({ - "name": parsed_names_origins[key]['girl'][name], - "origin": key, - "gender": "girl", - "matched": parsed_names_origins[key]['girl'][name], - "similar": "" - }) - } else { - var similarity = stringSimilarity.compareTwoStrings(req.body.string, parsed_names_origins[key]['girl'][name]); - if (similarity > 0.7) { - found.push({ - "name": req.body.string, - "origin": key, - "gender": "girl", - "matched": "", - "similar": parsed_names_origins[key]['girl'][name] - }) - } - } - } - for (name in parsed_names_origins[key]['uni']) { - if (req.body.string.includes(parsed_names_origins[key]['uni'][name])) { - found.push({ - "name": parsed_names_origins[key]['uni'][name], - "origin": key, - "gender": "uni", - "matched": parsed_names_origins[key]['uni'][name], - "similar": "" - }) - } else { - var similarity = stringSimilarity.compareTwoStrings(req.body.string, parsed_names_origins[key]['uni'][name]); - if (similarity > 0.7) { - found.push({ - "name": req.body.string, - "origin": key, - "gender": "uni", - "matched": "", - "similar": parsed_names_origins[key]['uni'][name] - }) - } - } - } - } +app.post("/analyze_string", async function(req, res, next) { - return found -} - -async function get_words_info(all_words, words_info) { - var temp_added = [] - for (let all_words_key of Object.keys(all_words)) { - for (let all_words_word of all_words[all_words_key]) { - if (!temp_added.includes(all_words_word)) { - temp_added.push(all_words_word); - var temp_words_info = { - "word": all_words_word, - "text": "", - "results": [] - } - try { - var url1 = "https://api.duckduckgo.com/?q={0}&format=json&pretty=1&no_html=1&skip_disambig=1".replace("{0}", all_words_word); - var url2 = "https://api.duckduckgo.com/?q={0}&format=json&pretty=1".replace("{0}", all_words_word); - var response1 = await get_url_wrapper_json(url1); - var response2 = await get_url_wrapper_json(url2); - if (response2.data != '') { - if ("RelatedTopics" in response2.data) { - if (response2.data.RelatedTopics.length > 0) { - if (response2.data != '') { - if ("AbstractText" in response1.data && response1.data.AbstractText != "") { - temp_words_info.text = response1.data.AbstractText; - } else if ("Abstract" in response1.data && response1.data.AbstractText != "") { - temp_words_info.text = response1.data.Abstract; - } else { - temp_words_info.text = "unknown"; - } - } - response2.data.RelatedTopics.forEach(function(item) { - if ("Name" in item) { - item.Topics.forEach(function(topic) { - temp_words_info.results.push({ - "type": item.Name, - "text": topic.Text, - "url": topic.FirstURL - }); - }); - } else { - temp_words_info.results.push({ - "type": "Related", - "text": item.Text, - "url": item.FirstURL - }); - } - }); - } - } - } - - if (temp_words_info.results.length > 0) { - words_info.push(temp_words_info); - } - } catch (error) { - verbose && console.log(error); - } - } - } - } -} - -async function check_engines(req, info) { - try { - if (google_api_key == "" || google_api_cs == "") { - return - } - var url = "https://www.googleapis.com/customsearch/v1?key={0}&cx={1}&q={2}".replace("{0}", google_api_key).replace("{1}", google_api_cs).replace("{2}", req.body.string); - var response = await get_url_wrapper_json(url); - if (response.data != '') { - try { - info.original = response.data.queries.request[0].searchTerms - } catch (e) {} - try { - info.corrected = response.data.spelling.correctedQuery - } catch (e) {} - try { - info.total = response.data.searchInformation.totalResults - } catch (e) {} - try { - response.data.items.forEach(function(item) { - info["items"].push({ - "title": item.title, - "snippet": item.snippet - }); - }); - } catch (e) {} - try { - if (info.total == 0 && info.corrected != "") { - info.checking = info.original + " [Error]
Try this: " + info.corrected; - } else if (info.total > 0 && info.corrected != "") { - info.checking = info.original + " [Good]
Suggested word: " + info.corrected + "
Total lookups: " + info.total; - } else if (info.total > 0 && info.corrected == "") { - info.checking = info.original + " [Good]
Total lookups: " + info.total; - } else { - info.checking = "Using " + info.original + " with no lookups"; - } - } catch (e) {} - } - } catch (error) { - verbose && console.log(error); - } -} - -function most_common1(all_words, temp_words) { - var temp_list = [] - Object.keys(all_words).forEach(function(key) { - all_words[key].forEach(function(item) { - if (!temp_list.includes(item) && item.length > 1) { - temp_list.push(item); - var temp = findWord(item); - if (Object.keys(temp).length != 0) { - var languages = Object.keys(temp).map(function(key) { - return [key, temp[key]]; - }); - languages.sort(function(first, second) { - return second[1] - first[1] - }).reverse(); - temp_words.push({ - "word": item, - "languages": languages.map(e => e.join(":")).join(" ") - }); - } - } - }); - }); -} - -async function most_common(all_words, temp_words) { - var temp_list = [] - Object.keys(all_words).forEach(function(key) { - all_words[key].forEach(function(item) { - if (!temp_list.includes(item) && item.length > 1) { - temp_list.push(item); - var temp = findWord(item); - if (Object.keys(temp).length != 0) { - var languages = Object.keys(temp).map(function(key) { - return [key, temp[key]]; - }); - languages.sort(function(first, second) { - return second[1] - first[1] - }).reverse(); - temp_words.push({ - "word": item, - "languages": languages.map(e => e[0]).join(", ") - }); - } - } - }); - }); -} - -function find_other(all_words) { - var words = WordsNinja.splitSentence(req.body.string); - - words.forEach(function(word) { - var value = false - Object.keys(all_words).forEach(function(key) { - if (all_words[key].includes(word)) { - value = true - } - }); - - if (!value && !all_words.maybe.includes(word)) { - all_words.maybe.push(word); - } - }); -} - -function remove_word(str, sub_string) { - part1 = str.substring(0, str.indexOf(sub_string)); - part2 = str.substring(str.indexOf(sub_string) + sub_string.length, str.length); - temp = (part1 + part2).replace(/[ \[\]:"\\|,.<>\/?~`!@#$%^&*()_+\-={};"]/gi, ""); - return temp; -} - -async function analyze_name(req, all_words) { - log_to_file_queue(req.body.uuid, "[Starting] String analysis") - temp_rr_names = [] - string_to_check = req.body.string - parsed_json.prefix.forEach(function(item, index) { - if (string_to_check.indexOf(item) == 0 && !all_words.prefix.includes(item)) { - all_words.prefix.push(item); - temp = remove_word(string_to_check, item); - if (temp !== null && temp !== "" && !all_words.unknown.includes(temp) && !all_words.maybe.includes(temp) && temp.length > 1) { - all_words.unknown.push(temp); - } - } - }); - parsed_json.m_names.forEach(function(item, index) { - if (string_to_check.indexOf(item) >= 0 && !all_words.name.includes(item)) { - all_words.name.push(item); - temp = remove_word(string_to_check, item); - if (temp !== null && temp !== "" && !all_words.unknown.includes(temp) && !all_words.maybe.includes(temp) && temp.length > 1) { - all_words.unknown.push(temp); - } - } - }); - parsed_json.f_names.forEach(function(item, index) { - if (string_to_check.indexOf(item) >= 0 && !all_words.name.includes(item)) { - all_words.name.push(item); - temp = remove_word(string_to_check, item); - if (temp !== null && temp !== "" && !all_words.unknown.includes(temp) && !all_words.maybe.includes(temp) && temp.length > 1) { - all_words.unknown.push(temp); - } - } - }); - - all_words.prefix.forEach(function(h_item, index) { - all_words.unknown.forEach(function(r_item, index) { - if (r_item.indexOf(h_item) == 0) { - temp = remove_word(r_item, h_item); - if (temp !== null && temp !== "" && !temp_rr_names.includes(temp) && !all_words.maybe.includes(temp) && temp.length > 1) { - temp_rr_names.push(temp); - } - } - }); - }); - - var temp_r_concat = all_words.unknown.concat(temp_rr_names.filter((item) => all_words.unknown.indexOf(item) < 0)); - - all_words.unknown = temp_r_concat - temp_rr_names = [] - - all_words.number.forEach(function(n_item, index) { - all_words.unknown.forEach(function(r_item, index) { - if (r_item.indexOf(n_item) >= 0) { - temp = remove_word(r_item, n_item); - if (temp !== null && temp !== "" && !temp_rr_names.includes(temp) && !all_words.maybe.includes(temp) && temp.length > 1) { - temp_rr_names.push(temp); - } - } - }); - }); - - var temp_r_concat = all_words.unknown.concat(temp_rr_names.filter((item) => all_words.unknown.indexOf(item) < 0)); - all_words.unknown = temp_r_concat - log_to_file_queue(req.body.uuid, "[Done] String analysis") -} - -app.post("/url", async function(req, res, next) { - await WordsNinja.loadDictionary(); var info = { "items": [], "original": "", @@ -1210,14 +224,14 @@ app.post("/url", async function(req, res, next) { } else { req.body.uuid = req.body.uuid.replace(/[^a-zA-Z0-9\-]+/g, ''); if (req.body.option.includes("FindUserProfilesSpecial")) { - log_to_file_queue(req.body.uuid, "[Starting] Checking user profiles special") - user_info_special.data = await find_username_special(req); - log_to_file_queue(req.body.uuid, "[Done] Checking user profiles special") + helper.log_to_file_queue(req.body.uuid, "[Starting] Checking user profiles special") + user_info_special.data = await special.find_username_special(req); + helper.log_to_file_queue(req.body.uuid, "[Done] Checking user profiles special") } if (req.body.option.includes("FindUserProfilesFast")) { - log_to_file_queue(req.body.uuid, "[Starting] Checking user profiles normal") - user_info_advanced.data = await find_username_normal(req); - log_to_file_queue(req.body.uuid, "[Done] Checking user profiles normal") + helper.log_to_file_queue(req.body.uuid, "[Starting] Checking user profiles normal") + user_info_advanced.data = await fastscan.find_username_normal(req); + helper.log_to_file_queue(req.body.uuid, "[Done] Checking user profiles normal") } if (req.body.option.includes("FindUserProfilesSlow") || req.body.option.includes("ShowUserProfilesSlow")) { if (!req.body.option.includes("FindUserProfilesSlow")) { @@ -1225,73 +239,51 @@ app.post("/url", async function(req, res, next) { } else if (!req.body.option.includes("ShowUserProfilesSlow")) { user_info_normal.type = "noshow" } - log_to_file_queue(req.body.uuid, "[Starting] Checking user profiles advanced") - user_info_normal.data = await find_username_advanced(req); - log_to_file_queue(req.body.uuid, "[Done] Checking user profiles advanced") + helper.log_to_file_queue(req.body.uuid, "[Starting] Checking user profiles advanced") + user_info_normal.data = await slowscan.find_username_advanced(req); + helper.log_to_file_queue(req.body.uuid, "[Done] Checking user profiles advanced") } if (req.body.option.includes("LookUps")) { - log_to_file_queue(req.body.uuid, "[Starting] Lookup") - await check_engines(req, info); - log_to_file_queue(req.body.uuid, "[Done] Lookup") + helper.log_to_file_queue(req.body.uuid, "[Starting] Lookup") + await externalapis.check_engines(req, info); + helper.log_to_file_queue(req.body.uuid, "[Done] Lookup") } if (req.body.option.includes("CustomSearch")) { - log_to_file_queue(req.body.uuid, "[Starting] Custom Search") - custom_search = await custom_search_ouputs(req); - log_to_file_queue(req.body.uuid, "[Done] Custom Search") + helper.log_to_file_queue(req.body.uuid, "[Starting] Custom Search") + custom_search = await externalapis.custom_search_ouputs(req); + helper.log_to_file_queue(req.body.uuid, "[Done] Custom Search") + } + if (req.body.option.includes("FindOrigins")) { + helper.log_to_file_queue(req.body.uuid, "[Starting] Finding Origins") + names_origins = await nameanalysis.find_origins(req); + helper.log_to_file_queue(req.body.uuid, "[Done] Finding Origins") } if (req.body.option.includes("SplitWordsByUpperCase")) { - try { - req.body.string.match(/[A-Z][a-z]+/g).forEach((item) => { - if (item.length > 1 && !all_words.unknown.includes(item) && !all_words.maybe.includes(item)) { - all_words.unknown.push(item.toLowerCase()); - } - }); - log_to_file_queue(req.body.uuid, "[Done] Split by UpperCase") - } catch (err) {} + helper.log_to_file_queue(req.body.uuid, "[Starting] Split by UpperCase") + await stringanalysis.split_upper_case(req,all_words) + helper.log_to_file_queue(req.body.uuid, "[Done] Split by UpperCase") } - if (req.body.option.includes("SplitWordsByAlphabet")) { - try { - req.body.string.match(/[A-Za-z]+/g).forEach((item) => { - if (item.length > 1 && !all_words.unknown.includes(item) && !all_words.maybe.includes(item)) { - all_words.unknown.push(item.toLowerCase()); - } - }); - log_to_file_queue(req.body.uuid, "[Done] Split by Alphabet") - } catch (err) {} + helper.log_to_file_queue(req.body.uuid, "[Starting] Split by Alphabet") + await stringanalysis.split_alphabet_case(req,all_words) + helper.log_to_file_queue(req.body.uuid, "[Done] Split by Alphabet") } - - if (req.body.option.includes("FindOrigins")) { - log_to_file_queue(req.body.uuid, "[Starting] Finding Origins") - names_origins = await find_origins(req); - log_to_file_queue(req.body.uuid, "[Done] Finding Origins") + if (req.body.option.includes("FindSymbols")) { + helper.log_to_file_queue(req.body.uuid, "[Starting] Finding Symbols") + await stringanalysis.find_symbols(req,all_words) + helper.log_to_file_queue(req.body.uuid, "[Done] Finding Symbols") + } + if (req.body.option.includes("FindNumbers")) { + helper.log_to_file_queue(req.body.uuid, "[Starting] Finding Numbers") + await stringanalysis.find_numbers(req,all_words) + helper.log_to_file_queue(req.body.uuid, "[Done] Finding Numbers") } - req.body.string = req.body.string.toLowerCase(); if (req.body.option.includes("ConvertNumbers")) { - numbers_to_letters = { - "4": "a", - "8": "b", - "3": "e", - "1": "l", - "0": "o", - "5": "s", - "7": "t", - "2": "z" - } - - temp_value = "" - for (i = 0; i < req.body.string.length; i++) { - _temp = numbers_to_letters[req.body.string.charAt(i)] - if (_temp != undefined) { - temp_value += numbers_to_letters[req.body.string.charAt(i)]; - } else { - temp_value += req.body.string.charAt(i); - } - } - req.body.string = temp_value - log_to_file_queue(req.body.uuid, "[Done] Convert numbers to letters") + helper.log_to_file_queue(req.body.uuid, "[Starting] Convert Numbers") + await stringanalysis.convert_numbers(req,all_words) + helper.log_to_file_queue(req.body.uuid, "[Done] Convert Numbers") } if (req.body.option.includes("LookUps") || @@ -1302,47 +294,20 @@ app.post("/url", async function(req, res, next) { req.body.option.includes("FindSymbols") || req.body.option.includes("FindNumbers") || req.body.option.includes("ConvertNumbers")) { - if (req.body.option.includes("FindNumbers")) { - try { - req.body.string.match(/(\d+)/g).forEach((item) => { - if (!all_words.number.includes(item)) { - all_words.number.push(item); - } - }); - } catch (err) {} - } - if (req.body.option.includes("FindSymbols")) { - try { - req.body.string.match(/[ \[\]:"\\|,.<>\/?~`!@#$%^&*()_+\-={};']/gi).forEach((item) => { - if (item !== " " && !all_words.symbol.includes(item)) { - all_words.symbol.push(item); - } - }); - } catch (err) {} - } + await stringanalysis.get_maybe_words(req, all_words) + await stringanalysis.analyze_string(req, all_words); - if (req.body.option.includes("SplitUpperCase")) { - req.body.string = req.body.string.replace(/([A-Z]+)/g, " $1"); - if (req.body.string.startsWith(" ")) { - req.body.string = req.body.string.substring(1); - } - } - all_words.maybe = WordsNinja.splitSentence(req.body.string).filter(function(elem, index, self) { - return index === self.indexOf(elem); - }).filter(word => word.length > 1); - await analyze_name(req, all_words); - //find_other(all_words) Object.keys(all_words).forEach((key) => (all_words[key].length == 0) && delete all_words[key]); if (req.body.option.includes("MostCommon")) { - await most_common(all_words, temp_words); + await stringanalysis.most_common(all_words, temp_words); } if (req.body.option.includes("WordInfo")) { - await get_words_info(all_words, words_info); + await externalapis.get_words_info(all_words, words_info); } } else if (req.body.option.includes("NormalAnalysis@@")) { - var maybe_words = WordsNinja.splitSentence(req.body.string); + //var maybe_words = WordsNinja.splitSentence(req.body.string); all_words.maybe = maybe_words.filter(function(elem, index, self) { return index === self.indexOf(elem); }); @@ -1373,84 +338,101 @@ app.post("/url", async function(req, res, next) { }); app.use((err, req, res, next) => { - verbose && console.log(" --- Global Error ---") - verbose && console.log(pe.render(err)); + helper.verbose && console.log(" --- Global Error ---") + helper.verbose && console.log(pe.render(err)); res.json("Error"); }); process.on('uncaughtException', function(err) { - verbose && console.log(" --- Uncaught Error ---") - verbose && console.log(pe.render(err)); + helper.verbose && console.log(" --- Uncaught Error ---") + helper.verbose && console.log(pe.render(err)); }) process.on('unhandledRejection', function(err) { - verbose && console.log(" --- Uncaught Rejection ---") - verbose && console.log(pe.render(err)); + helper.verbose && console.log(" --- Uncaught Rejection ---") + helper.verbose && console.log(pe.render(err)); }) const server_host = '0.0.0.0'; const server_port = process.env.PORT || 9005; - -async function check_user_cli(username, websites) { +async function check_user_cli(argv) { var ret = [] var random_string = Math.random().toString(36).substring(2); var req = { 'body': { 'uuid': random_string, - 'string': username, - 'option': 'FindUserProfilesFast' + 'string': argv.username, + 'option': 'FindUserProfilesFast,' + argv.output } } - await parsed_sites.forEach(async function(value, i) { - parsed_sites[i].selected = "false" - if (websites.length > 0) { - await websites.split(' ').forEach(item => { - if (parsed_sites[i].url.toLowerCase().includes(item.toLowerCase())) { - parsed_sites[i].selected = "true" + await helper.parsed_sites.forEach(async function(value, i) { + helper.parsed_sites[i].selected = "false" + if (argv.websites.length > 0) { + await argv.websites.split(' ').forEach(item => { + if (helper.parsed_sites[i].url.toLowerCase().includes(item.toLowerCase())) { + helper.parsed_sites[i].selected = "true" } }); } }); - ret = await find_username_normal(req) + + ret = await fastscan.find_username_normal(req) if (typeof ret === 'undefined' || ret === undefined || ret.length == 0) { - log_to_file_queue(req.body.uuid, 'User does not exist (try FindUserProfilesSlow or FindUserProfilesSpecial)'); + helper.log_to_file_queue(req.body.uuid, 'User does not exist (try FindUserProfilesSlow or FindUserProfilesSpecial)'); } else { + var temp_detected = [] await ret.forEach(item => { - delete item['title'] - delete item['image'] - delete item['text'] - item['link'] = get_site_from_url(item['link']) - log_to_file_queue(req.body.uuid, item); + var temp_keys = {"found": 0,"link": "","rate": "","title": "","text": "","image":""}; + if (argv.options == "") + { + delete item['text'] + delete item['image'] + } + else{ + for (var key in temp_keys) { + if (!argv.options.includes(key)){ + delete item[key] + } + } + } + + temp_detected.push(item) + + if (argv.output == "pretty"){ + helper.log_to_file_queue(req.body.uuid, item); + } + else if (argv.output == ""){ + helper.log_to_file_queue(req.body.uuid, item); + } }); + + if (argv.output == "json"){ + console.log(JSON.stringify(temp_detected, null, 2)) + } } }; async function list_all_websites() { var temp_arr = [] - await parsed_sites.forEach(item => { - temp_arr.push(get_site_from_url(item.url)) + await helper.parsed_sites.forEach(item => { + temp_arr.push(helper.get_site_from_url(item.url)) }); + console.log('[Listing] Available websites\n' + temp_arr.join('\n')) } -if ('grid' in argv) { - grid_url = argv.grid +if (argv.grid != "") { + helper.grid_url = argv.grid } -if ('cli' in argv) { - if ('list' in argv) { +if (argv.cli) { + if (argv.list) { list_all_websites(); - } else if ('mode' in argv) { - if (argv.mode == 'fast') { - if ('user' in argv && 'website' in argv) { - if (argv.user != "" && argv.website != "") { - check_user_cli(argv.user, argv.website) - } else { - console.log("user or website is empty, use -h for help") - } - } + } else if (argv.mode == "fast") { + if (argv.usernmae != "" && argv.websites != "") { + check_user_cli(argv) } } } else { diff --git a/info b/info index 5e76488..c8c00ba 100644 --- a/info +++ b/info @@ -1,9 +1,9 @@ -{"version":"2021.V.2.09", +{"version":"2021.V.2.10", "build":"pass", "test":"pass", "grid_test":"pass", "websites":"342", "detections":"939", - "special":"2", + "special":"3", "awaiting_verification":"22", - "auto_testing":"606e2f07-cf0d-4f05-98e0-bfd6ed204a1a"} + "auto_testing":"bb581703-6333-4d78-bb30-e20a9e846145"} diff --git a/modules/externalapis.js b/modules/externalapis.js new file mode 100644 index 0000000..51408ac --- /dev/null +++ b/modules/externalapis.js @@ -0,0 +1,153 @@ +var helper = require("./helper.js") +var async = require("async"); + +async function get_words_info(all_words, words_info) { + var temp_added = [] + for (let all_words_key of Object.keys(all_words)) { + for (let all_words_word of all_words[all_words_key]) { + if (!temp_added.includes(all_words_word)) { + temp_added.push(all_words_word); + var temp_words_info = { + "word": all_words_word, + "text": "", + "results": [] + } + try { + var url1 = "https://api.duckduckgo.com/?q={0}&format=json&pretty=1&no_html=1&skip_disambig=1".replace("{0}", all_words_word); + var url2 = "https://api.duckduckgo.com/?q={0}&format=json&pretty=1".replace("{0}", all_words_word); + var response1 = await helper.get_url_wrapper_json(url1); + var response2 = await helper.get_url_wrapper_json(url2); + if (response2.data != '') { + if ("RelatedTopics" in response2.data) { + if (response2.data.RelatedTopics.length > 0) { + if (response2.data != '') { + if ("AbstractText" in response1.data && response1.data.AbstractText != "") { + temp_words_info.text = response1.data.AbstractText; + } else if ("Abstract" in response1.data && response1.data.AbstractText != "") { + temp_words_info.text = response1.data.Abstract; + } else { + temp_words_info.text = "unknown"; + } + } + response2.data.RelatedTopics.forEach(function(item) { + if ("Name" in item) { + item.Topics.forEach(function(topic) { + temp_words_info.results.push({ + "type": item.Name, + "text": topic.Text, + "url": topic.FirstURL + }); + }); + } else { + temp_words_info.results.push({ + "type": "Related", + "text": item.Text, + "url": item.FirstURL + }); + } + }); + } + } + } + + if (temp_words_info.results.length > 0) { + words_info.push(temp_words_info); + } + } catch (error) { + helper.verbose && console.log(error); + } + } + } + } +} + +async function check_engines(req, info) { + try { + if (helper.google_api_key == "" || helper.google_api_cs == "") { + return + } + var url = "https://www.googleapis.com/customsearch/v1?key={0}&cx={1}&q={2}".replace("{0}", helper.google_api_key).replace("{1}", helper.google_api_cs).replace("{2}", req.body.string); + var response = await helper.get_url_wrapper_json(url); + if (response.data != '') { + try { + info.original = response.data.queries.request[0].searchTerms + } catch (e) {} + try { + info.corrected = response.data.spelling.correctedQuery + } catch (e) {} + try { + info.total = response.data.searchInformation.totalResults + } catch (e) {} + try { + response.data.items.forEach(function(item) { + info["items"].push({ + "title": item.title, + "snippet": item.snippet + }); + }); + } catch (e) {} + try { + if (info.total == 0 && info.corrected != "") { + info.checking = info.original + " [Error]
Try this: " + info.corrected; + } else if (info.total > 0 && info.corrected != "") { + info.checking = info.original + " [Good]
Suggested word: " + info.corrected + "
Total lookups: " + info.total; + } else if (info.total > 0 && info.corrected == "") { + info.checking = info.original + " [Good]
Total lookups: " + info.total; + } else { + info.checking = "Using " + info.original + " with no lookups"; + } + } catch (e) {} + } + } catch (error) { + helper.verbose && console.log(error); + } +} + +async function custom_search_ouputs(req) { + var possible_parameters = ['user', 'profile', 'account'] + const time = new Date(); + const functions = []; + possible_parameters.forEach((key) => { + functions.push(custom_search_ouputs_website.bind(null, req.body.uuid, req.body.string, key)); + }); + const results = await async.parallelLimit(functions, 6); + helper.verbose && console.log(`Total time ${new Date() - time}`); + var merged = [].concat.apply([], results.filter(item => item !== undefined)); + return merged +} + +async function custom_search_ouputs_website(uuid, name, key) { + + return new Promise(async (resolve, reject) => { + try { + var results = [] + helper.log_to_file_queue(uuid, "[Custom Search Using] " + key) + var url = "https://www.googleapis.com/customsearch/v1?key={0}&cx={1}&q={2}:{3}".replace("{0}", helper.google_api_key).replace("{1}", helper.google_api_cs).replace("{2}", key).replace("{3}", name); + var response = await helper.get_url_wrapper_json(url); + if (response.data != '') { + if ('items' in response.data) { + response.data.items.forEach((key) => { + results.push({ + site: helper.get_site_from_url(key.link), + link: key.link, + snippet: key.snippet + }) + }); + } + } + if (results.length > 0) { + resolve(results); + } else { + resolve(undefined) + } + } catch (err) { + resolve(undefined) + } + }); +} + +module.exports = { + check_engines, + get_words_info, + custom_search_ouputs +} diff --git a/modules/fastscan.js b/modules/fastscan.js new file mode 100644 index 0000000..545d145 --- /dev/null +++ b/modules/fastscan.js @@ -0,0 +1,98 @@ +var helper = require("./helper.js") +var async = require("async"); +var sanitizeHtml = require("sanitize-html"); +const { + htmlToText +} = require('html-to-text'); +var cheerio = require('cheerio'); + +async function find_username_normal(req) { + const time = new Date(); + const functions = []; + await helper.parsed_sites.forEach(site => { + if ("status" in site) { + if (site.status == "bad") { + return Promise.resolve(); + } + } + if (site.selected == "true" && site.detections.length > 0) { + functions.push(find_username_site.bind(null, req.body.uuid, req.body.string, req.body.option, site)); + } + }); + const results = await async.parallelLimit(functions, 100); + helper.verbose && console.log(`Total time ${new Date() - time}`); + return results.filter(item => item !== undefined) +} + +async function find_username_site(uuid, username, options, site) { + return new Promise(async (resolve, reject) => { + try { + if (!options.includes('json')) { + helper.log_to_file_queue(uuid, "[Checking] " + helper.get_site_from_url(site.url)) + } + var body = await helper.get_url_wrapper_text(site.url.replace("{username}", username)); + var detections_count = 0; + var source = body; + var text_only = "unavailable"; + var title = "unavailable"; + var temp_profile = { + "found": 0, + "image": "", + "link": "", + "rate": "", + "title": "", + "text": "", + "type": "" + }; + + await Promise.all(site.detections.map(async detection => { + var temp_found = "false"; + if (detection.type == "normal" && options.includes("FindUserProfilesFast") && source != "") { + detections_count += 1 + if (source.toLowerCase().includes(detection.string.replace("{username}", username).toLowerCase())) { + temp_found = "true"; + } + if (detection.return == temp_found) { + //console.log(detection.string, " > normal"); + temp_profile.found += 1 + } + } + })); + if (temp_profile.found > 0 && detections_count != 0) { + temp_profile.text = sanitizeHtml(htmlToText(body, { + wordwrap: false, + hideLinkHrefIfSameAsText: true, + ignoreHref: true, + ignoreImage: true + })); + if (temp_profile.text == "") { + temp_profile.text = "unavailable" + } + + try { + var $ = cheerio.load(body); + title = sanitizeHtml($("title").text()) + if (title.length == 0) { + title = "unavailable" + } + } catch (err) { + helper.verbose && console.log(err); + } + + temp_profile.title = title; + temp_profile.rate = "%" + ((temp_profile["found"] / detections_count) * 100).toFixed(2); + temp_profile.link = site.url.replace("{username}", username); + temp_profile.type = site.type + resolve(temp_profile); + } else { + resolve(undefined) + } + } catch (err) { + resolve(undefined) + } + }); +} + +module.exports = { + find_username_normal +} diff --git a/modules/helper.js b/modules/helper.js new file mode 100644 index 0000000..6e37f7f --- /dev/null +++ b/modules/helper.js @@ -0,0 +1,112 @@ +var verbose = false +var google_api_key = ""; +var google_api_cs = ""; +var grid_url = ""; +var proxy = "" + +var header_options = { + headers: { + 'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:84.0) Gecko/20100101 Firefox/84.0', + } +}; + +var https = require("follow-redirects").https; +var fs = require("fs"); +var url = require("url"); + +var parsed_sites = JSON.parse(fs.readFileSync("sites.json")); +var logs_queue = Promise.resolve(); + +function log_to_file_queue(uuid, msg) { + logs_queue = logs_queue.then(function() { + return new Promise(function(resolve) { + fs.appendFile("logs/" + uuid + "_log.txt", msg + "\n", function(err, data) { + console.log(msg) + resolve(); + }); + }); + }); +} + +function get_site_from_url(_url) { + temp = url.parse(_url.replace("{username}", "nothinghere")).hostname + return temp.replace("nothinghere.", "") +} + +async function get_url_wrapper_json(url, time = 5) { + try { + let http_promise = new Promise((resolve, reject) => { + var request = https.get(url, header_options, function(res) { + var body = "" + res.on("data", function(chunk) { + body += chunk; + }); + res.on("end", function() { + resolve({ + 'data': JSON.parse(body.toString()) + }); + }); + }); + request.on('error', function(e) { + reject({ + 'data': '' + }) + }); + request.on('socket', function(socket) { + var timeout = (time != 0) ? time * 1000 : 5000; + socket.setTimeout(timeout, function() { + request.abort(); + }); + }); + }); + let response_body = await http_promise; + return response_body + } catch (err) { + verbose && console.log(err); + } +} + +async function get_url_wrapper_text(url, time = 5) { + try { + let http_promise = new Promise((resolve, reject) => { + var request = https.get(url, header_options, function(res) { + var body = "" + res.on("data", function(chunk) { + body += chunk; + }); + res.on("end", function() { + resolve(body); + }); + }); + request.on('error', function(e) { + reject({ + 'data': '' + }) + }); + request.on('socket', function(socket) { + var timeout = (time != 0) ? time * 1000 : 5000; + socket.setTimeout(timeout, function() { + request.abort(); + }); + }); + }); + let response_body = await http_promise; + return response_body + } catch (err) { + verbose && console.log(err); + } +} + +module.exports = { + parsed_sites, + verbose, + google_api_key, + google_api_cs, + grid_url, + header_options, + proxy, + get_site_from_url, + log_to_file_queue, + get_url_wrapper_text, + get_url_wrapper_json +} diff --git a/modules/nameanalysis.js b/modules/nameanalysis.js new file mode 100644 index 0000000..bd0b4a9 --- /dev/null +++ b/modules/nameanalysis.js @@ -0,0 +1,83 @@ +var helper = require("./helper.js") +var fs = require("fs"); +var stringSimilarity = require("string-similarity"); + +var parsed_names_origins = JSON.parse(fs.readFileSync("names.json")); + +async function find_origins(req) { + var found = [] + for (key in parsed_names_origins) { + for (name in parsed_names_origins[key]['boy']) { + if (req.body.string.includes(parsed_names_origins[key]['boy'][name])) { + found.push({ + "name": parsed_names_origins[key]['boy'][name], + "origin": key, + "gender": "boy", + "matched": parsed_names_origins[key]['boy'][name], + "similar": "" + }) + } else { + var similarity = stringSimilarity.compareTwoStrings(req.body.string, parsed_names_origins[key]['boy'][name]); + if (similarity > 0.7) { + found.push({ + "name": req.body.string, + "origin": key, + "gender": "boy", + "matched": "", + "similar": parsed_names_origins[key]['boy'][name] + }) + } + } + } + for (name in parsed_names_origins[key]['girl']) { + if (req.body.string.includes(parsed_names_origins[key]['girl'][name])) { + found.push({ + "name": parsed_names_origins[key]['girl'][name], + "origin": key, + "gender": "girl", + "matched": parsed_names_origins[key]['girl'][name], + "similar": "" + }) + } else { + var similarity = stringSimilarity.compareTwoStrings(req.body.string, parsed_names_origins[key]['girl'][name]); + if (similarity > 0.7) { + found.push({ + "name": req.body.string, + "origin": key, + "gender": "girl", + "matched": "", + "similar": parsed_names_origins[key]['girl'][name] + }) + } + } + } + for (name in parsed_names_origins[key]['uni']) { + if (req.body.string.includes(parsed_names_origins[key]['uni'][name])) { + found.push({ + "name": parsed_names_origins[key]['uni'][name], + "origin": key, + "gender": "uni", + "matched": parsed_names_origins[key]['uni'][name], + "similar": "" + }) + } else { + var similarity = stringSimilarity.compareTwoStrings(req.body.string, parsed_names_origins[key]['uni'][name]); + if (similarity > 0.7) { + found.push({ + "name": req.body.string, + "origin": key, + "gender": "uni", + "matched": "", + "similar": parsed_names_origins[key]['uni'][name] + }) + } + } + } + } + + return found +} + +module.exports = { + find_origins +} diff --git a/modules/slowscan.js b/modules/slowscan.js new file mode 100644 index 0000000..8a8dca2 --- /dev/null +++ b/modules/slowscan.js @@ -0,0 +1,182 @@ +var helper = require("./helper.js") +var async = require("async"); +var sanitizeHtml = require("sanitize-html"); +var tesseract = require("node-tesseract-ocr"); +var sanitizeHtml = require("sanitize-html"); +var firefox = require("selenium-webdriver/firefox"); +var { + Builder, + By, + Key +} = require("selenium-webdriver"); +var tmp = require("tmp"); +var fs = require("fs"); + +async function find_username_advanced(req) { + const time = new Date(); + const functions = []; + helper.parsed_sites.forEach((site) => { + if ("status" in site) { + if (site.status == "bad") { + return Promise.resolve(); + } + } + if (site.selected == "true" && site.detections.length > 0) { + functions.push(find_username_site.bind(null, req.body.uuid, req.body.string, req.body.option, site)); + } + }); + const results = await async.parallelLimit(functions, 8); + helper.verbose && console.log(`Total time ${new Date() - time}`); + return results.filter(item => item !== undefined) +} + +async function find_username_site(uuid, username, options, site) { + return new Promise(async (resolve, reject) => { + helper.log_to_file_queue(uuid, "[Checking] " + helper.get_site_from_url(site.url)) + let driver = undefined + if (helper.grid_url == "") { + driver = new Builder() + .forBrowser("firefox") + .setFirefoxOptions(new firefox.Options().headless().windowSize({ + width: 640, + height: 480 + })) + .build(); + } else { + driver = new Builder() + .forBrowser("firefox") + .setFirefoxOptions(new firefox.Options().headless().windowSize({ + width: 640, + height: 480 + })) + .usingServer(helper.grid_url) + .build(); + } + + try { + + var timeouts = { + implicit: 0, + pageLoad: 5000, + script: 5000 + }; + + var timeout = (site.timeout != 0) ? site.timeout * 1000 : 5000; + var implicit = (site.implicit != 0) ? site.implicit * 1000 : 0; + + timeouts = { + implicit: implicit, + pageLoad: timeout, + script: timeout + }; + + helper.verbose && console.log(timeouts) + + var source = ""; + var data = ""; + var text_only = "unavailable"; + var title = "unavailable"; + var temp_profile = { + "found": 0, + "image": "", + "link": "", + "rate": "", + "title": "", + "text": "", + "type": "" + }; + var link = site.url.replace("{username}", username); + await driver.manage().setTimeouts(timeouts); + await driver.get(link);; + source = await driver.getPageSource(); + data = await driver.takeScreenshot(); + title = await driver.getTitle(); + text_only = await driver.findElement(By.tagName("body")).getText(); + await driver.quit() + if (options.includes("ShowUserProfilesSlow")) { + temp_profile["image"] = "data:image/png;base64,{image}".replace("{image}", data); + } + if (site.selected == "true" && site.detections.length > 0 && options.includes("FindUserProfilesSlow")) { + await Promise.all(site.detections.map(async detection => { + try { + if ("status" in detection) { + if (detection.status == "bad") { + return; + } + } + var temp_found = "false" + if (detection.type == "ocr" && data != "") { + tmpobj = tmp.fileSync(); + fs.writeFileSync(tmpobj.name, Buffer.from(data, "base64")); + await tesseract.recognize(tmpobj.name, { + lang: "eng", + oem: 1, + psm: 3, + }) + .then(text => { + text = text.replace(/[^A-Za-z0-9]/gi, ""); + detection.string = detection.string.replace(/[^A-Za-z0-9]/gi, ""); + if (text != "") { + if (text.toLowerCase().includes(detection.string.toLowerCase())) { + temp_found = "true"; + } + if (detection.return == temp_found) { + //console.log(text); + //console.log(detection.string," > Found ocr"); + temp_profile.found += 1; + } + } + }) + .catch(error => { + helper.verbose && console.log(error.message); + }) + tmpobj.removeCallback(); + } else if (detection.type == "normal" && source != "") { + if (source.toLowerCase().includes(detection.string.replace("{username}", username).toLowerCase())) { + temp_found = "true"; + } + if (detection.return == temp_found) { + //console.log(detection.string," > normal"); + temp_profile.found += 1 + } + } else if (detection.type == "advanced" && text_only != "") { + if (text_only.toLowerCase().includes(detection.string.replace("{username}", username).toLowerCase())) { + temp_found = "true"; + } + if (detection.return == temp_found) { + //console.log(detection.string," > normal"); + temp_profile.found += 1 + } + } + + } catch (err) { + + } + })); + } + if (temp_profile.found > 0 || temp_profile.image != "") { + temp_profile.text = sanitizeHtml(text_only); + temp_profile.title = sanitizeHtml(title); + temp_profile.rate = "%" + ((temp_profile.found / site.detections.length) * 100).toFixed(2); + temp_profile.link = site.url.replace("{username}", username); + temp_profile.type = site.type + resolve(temp_profile); + } else { + resolve(undefined) + } + } catch (err) { + if (driver !== undefined) { + try { + await driver.quit() + } catch (err) { + helper.verbose && console.log("Driver Session Issue") + } + } + resolve(undefined) + } + }); +} + +module.exports = { + find_username_advanced +} diff --git a/modules/special.js b/modules/special.js new file mode 100644 index 0000000..fbbfbba --- /dev/null +++ b/modules/special.js @@ -0,0 +1,241 @@ +var helper = require("./helper.js") +var async = require("async"); +var sanitizeHtml = require("sanitize-html"); +var tesseract = require("node-tesseract-ocr"); +var sanitizeHtml = require("sanitize-html"); +var firefox = require("selenium-webdriver/firefox"); +var { + Builder, + By, + Key +} = require("selenium-webdriver"); + +async function find_username_special(req) { + const time = new Date(); + const functions = []; + helper.parsed_sites.forEach((site) => { + if ("status" in site) { + if (site.status == "bad") { + return Promise.resolve(); + } + } + if (site.selected == "true") { + site.detections.forEach((detection) => { + if (detection.type == 'special') { + if (detection.function == 'special_facebook_1') { + functions.push(find_username_site_special_facebook_1.bind(null, req.body.uuid, req.body.string, site)); + } else if (detection.function == 'special_gmail_1') { + functions.push(find_username_site_special_gmail_1.bind(null, req.body.uuid, req.body.string, site)); + } else if (detection.function == 'special_google_1') { + functions.push(find_username_site_special_google_1.bind(null, req.body.uuid, req.body.string, site)); + } + } + }); + } + }); + const results = await async.parallelLimit(functions, 5); + helper.verbose && console.log(`Total time ${new Date() - time}`); + return results.filter(item => item !== undefined) +} + +async function find_username_site_special_facebook_1(uuid, username, site) { + return new Promise(async (resolve, reject) => { + helper.log_to_file_queue(uuid, "[Checking] " + helper.get_site_from_url(site.url)) + let driver = new Builder() + .forBrowser("firefox") + .setFirefoxOptions(new firefox.Options().headless().windowSize({ + width: 640, + height: 480 + })) + .build(); + + try { + var timeouts = { + implicit: 0, + pageLoad: 10000, + script: 10000 + }; + + var source = ""; + var data = ""; + var text_only = "unavailable"; + var title = "unavailable"; + var temp_profile = { + "found": 0, + "image": "", + "link": "", + "rate": "", + "title": "", + "text": "", + "type": "" + }; + var link = "https://mbasic.facebook.com/login/identify/?ctx=recoveqr"; + await driver.manage().setTimeouts(timeouts); + await driver.get(link);; + await driver.findElement(By.id('identify_search_text_input')).sendKeys(username); + await driver.findElement(By.id('did_submit')).click(); + source = await driver.getPageSource(); + text_only = await driver.findElement(By.tagName("body")).getText(); + await driver.quit() + if (source.includes("Try Entering Your Password")) { + temp_found = "true"; + temp_profile.found += 1 + } + if (temp_profile.found > 0) { + temp_profile.text = "unavailable"; + temp_profile.title = "unavailable"; + temp_profile.rate = "%" + ((temp_profile.found / 1) * 100).toFixed(2); + temp_profile.link = site.url.replace("{username}", username); + temp_profile.type = site.type + resolve(temp_profile); + } else { + resolve(undefined) + } + } catch (err) { + if (driver !== undefined) { + try { + await driver.quit() + } catch (err) { + helper.verbose && console.log("Driver Session Issue") + } + } + resolve(undefined) + } + }); +} + +async function find_username_site_special_gmail_1(uuid, username, site) { + return new Promise(async (resolve, reject) => { + helper.log_to_file_queue(uuid, "[Checking] " + helper.get_site_from_url(site.url)) + let driver = new Builder() + .forBrowser("firefox") + .setFirefoxOptions(new firefox.Options().headless().windowSize({ + width: 640, + height: 480 + })) + .build(); + + try { + var timeouts = { + implicit: 0, + pageLoad: 10000, + script: 10000 + }; + + var source = ""; + var data = ""; + var text_only = "unavailable"; + var title = "unavailable"; + var temp_profile = { + "found": 0, + "image": "", + "link": "", + "rate": "", + "title": "", + "text": "", + "type": "" + }; + var link = "https://accounts.google.com/signup/v2/webcreateaccount?service=mail&continue=https%3A%2F%2Fmail.google.com%2Fmail%2F%3Fpc%3Dtopnav-about-n-en&flowName=GlifWebSignIn&flowEntry=SignUp"; + await driver.manage().setTimeouts(timeouts); + await driver.get(link);; + await driver.findElement(By.id('username')).sendKeys(username); + await driver.findElement(By.id('selectioni1')).click(); + source = await driver.getPageSource(); + text_only = await driver.findElement(By.tagName("body")).getText(); + await driver.quit() + if (text_only.includes("That username is taken") && !text_only.includes('your username must be between') && !text_only.includes('You can use letters')) { + temp_found = "true"; + temp_profile.found += 1 + } + if (temp_profile.found > 0) { + temp_profile.text = username + "@gmail.com"; + temp_profile.title = "unavailable"; + temp_profile.rate = "%" + ((temp_profile.found / 1) * 100).toFixed(2); + temp_profile.link = "https://google.com" + temp_profile.type = site.type + resolve(temp_profile); + } else { + resolve(undefined) + } + } catch (err) { + if (driver !== undefined) { + try { + await driver.quit() + } catch (err) { + helper.verbose && console.log("Driver Session Issue") + } + } + resolve(undefined) + } + }); +} + +async function find_username_site_special_google_1(uuid, username, site) { + return new Promise(async (resolve, reject) => { + helper.log_to_file_queue(uuid, "[Checking] " + helper.get_site_from_url(site.url)) + let driver = new Builder() + .forBrowser("firefox") + .setFirefoxOptions(new firefox.Options().headless().windowSize({ + width: 640, + height: 480 + })) + .build(); + + try { + var timeouts = { + implicit: 0, + pageLoad: 10000, + script: 10000 + }; + + var source = ""; + var data = ""; + var text_only = "unavailable"; + var title = "unavailable"; + var temp_profile = { + "found": 0, + "image": "", + "link": "", + "rate": "", + "title": "", + "text": "", + "type": "" + }; + var link = "https://accounts.google.com/signin/v2/identifier?continue=https%3A%2F%2Fmail.google.com%2Fmail%2F&service=mail&sacu=1&rip=1&flowName=GlifWebSignIn&flowEntry=ServiceLogin"; + await driver.manage().setTimeouts(timeouts); + await driver.get(link);; + await driver.findElement(By.id('identifierId')).sendKeys(username); + await driver.findElement(By.xpath("//button[contains(.,'Next')]")).click(); + source = await driver.getPageSource(); + text_only = await driver.findElement(By.tagName("body")).getText(); + await driver.quit() + if (text_only.includes("Couldn't sign you in") && !text_only.includes("Couldn't find your")) { + temp_found = "true"; + temp_profile.found += 1 + } + if (temp_profile.found > 0) { + temp_profile.text = username + "@gmail.com"; + temp_profile.title = "unavailable"; + temp_profile.rate = "%" + ((temp_profile.found / 1) * 100).toFixed(2); + temp_profile.link = "https://google.com" + temp_profile.type = site.type + resolve(temp_profile); + } else { + resolve(undefined) + } + } catch (err) { + if (driver !== undefined) { + try { + await driver.quit() + } catch (err) { + helper.verbose && console.log("Driver Session Issue") + } + } + resolve(undefined) + } + }); +} + +module.exports = { + find_username_special +} diff --git a/modules/stringanalysis.js b/modules/stringanalysis.js new file mode 100644 index 0000000..17904ca --- /dev/null +++ b/modules/stringanalysis.js @@ -0,0 +1,229 @@ +var helper = require("./helper.js") +var { + findWord +} = require("most-common-words-by-language"); +var fs = require("fs"); +var parsed_json = JSON.parse(fs.readFileSync("dict.json")); +var WordsNinjaPack = require("wordsninja"); +var WordsNinja = new WordsNinjaPack(); + +function most_common1(all_words, temp_words) { + var temp_list = [] + Object.keys(all_words).forEach(function(key) { + all_words[key].forEach(function(item) { + if (!temp_list.includes(item) && item.length > 1) { + temp_list.push(item); + var temp = findWord(item); + if (Object.keys(temp).length != 0) { + var languages = Object.keys(temp).map(function(key) { + return [key, temp[key]]; + }); + languages.sort(function(first, second) { + return second[1] - first[1] + }).reverse(); + temp_words.push({ + "word": item, + "languages": languages.map(e => e.join(":")).join(" ") + }); + } + } + }); + }); +} + +async function most_common(all_words, temp_words) { + var temp_list = [] + Object.keys(all_words).forEach(function(key) { + all_words[key].forEach(function(item) { + if (!temp_list.includes(item) && item.length > 1) { + temp_list.push(item); + var temp = findWord(item); + if (Object.keys(temp).length != 0) { + var languages = Object.keys(temp).map(function(key) { + return [key, temp[key]]; + }); + languages.sort(function(first, second) { + return second[1] - first[1] + }).reverse(); + temp_words.push({ + "word": item, + "languages": languages.map(e => e[0]).join(", ") + }); + } + } + }); + }); +} + +function find_other(all_words) { + var words = WordsNinja.splitSentence(req.body.string); + + words.forEach(function(word) { + var value = false + Object.keys(all_words).forEach(function(key) { + if (all_words[key].includes(word)) { + value = true + } + }); + + if (!value && !all_words.maybe.includes(word)) { + all_words.maybe.push(word); + } + }); +} + +function remove_word(str, sub_string) { + part1 = str.substring(0, str.indexOf(sub_string)); + part2 = str.substring(str.indexOf(sub_string) + sub_string.length, str.length); + temp = (part1 + part2).replace(/[ \[\]:"\\|,.<>\/?~`!@#$%^&*()_+\-={};"]/gi, ""); + return temp; +} + +async function analyze_string(req, all_words) { + helper.log_to_file_queue(req.body.uuid, "[Starting] String analysis") + temp_rr_names = [] + string_to_check = req.body.string + parsed_json.prefix.forEach(function(item, index) { + if (string_to_check.indexOf(item) == 0 && !all_words.prefix.includes(item)) { + all_words.prefix.push(item); + temp = remove_word(string_to_check, item); + if (temp !== null && temp !== "" && !all_words.unknown.includes(temp) && !all_words.maybe.includes(temp) && temp.length > 1) { + all_words.unknown.push(temp); + } + } + }); + parsed_json.m_names.forEach(function(item, index) { + if (string_to_check.indexOf(item) >= 0 && !all_words.name.includes(item)) { + all_words.name.push(item); + temp = remove_word(string_to_check, item); + if (temp !== null && temp !== "" && !all_words.unknown.includes(temp) && !all_words.maybe.includes(temp) && temp.length > 1) { + all_words.unknown.push(temp); + } + } + }); + parsed_json.f_names.forEach(function(item, index) { + if (string_to_check.indexOf(item) >= 0 && !all_words.name.includes(item)) { + all_words.name.push(item); + temp = remove_word(string_to_check, item); + if (temp !== null && temp !== "" && !all_words.unknown.includes(temp) && !all_words.maybe.includes(temp) && temp.length > 1) { + all_words.unknown.push(temp); + } + } + }); + + all_words.prefix.forEach(function(h_item, index) { + all_words.unknown.forEach(function(r_item, index) { + if (r_item.indexOf(h_item) == 0) { + temp = remove_word(r_item, h_item); + if (temp !== null && temp !== "" && !temp_rr_names.includes(temp) && !all_words.maybe.includes(temp) && temp.length > 1) { + temp_rr_names.push(temp); + } + } + }); + }); + + var temp_r_concat = all_words.unknown.concat(temp_rr_names.filter((item) => all_words.unknown.indexOf(item) < 0)); + + all_words.unknown = temp_r_concat + temp_rr_names = [] + + all_words.number.forEach(function(n_item, index) { + all_words.unknown.forEach(function(r_item, index) { + if (r_item.indexOf(n_item) >= 0) { + temp = remove_word(r_item, n_item); + if (temp !== null && temp !== "" && !temp_rr_names.includes(temp) && !all_words.maybe.includes(temp) && temp.length > 1) { + temp_rr_names.push(temp); + } + } + }); + }); + + var temp_r_concat = all_words.unknown.concat(temp_rr_names.filter((item) => all_words.unknown.indexOf(item) < 0)); + all_words.unknown = temp_r_concat + helper.log_to_file_queue(req.body.uuid, "[Done] String analysis") +} + +async function split_upper_case(req, all_words) { + try { + req.body.string.match(/[A-Z][a-z]+/g).forEach((item) => { + if (item.length > 1 && !all_words.unknown.includes(item) && !all_words.maybe.includes(item)) { + all_words.unknown.push(item.toLowerCase()); + } + }); + } catch (err) {} +} + +async function split_alphabet_case(req, all_words) { + try { + req.body.string.match(/[A-Za-z]+/g).forEach((item) => { + if (item.length > 1 && !all_words.unknown.includes(item) && !all_words.maybe.includes(item)) { + all_words.unknown.push(item.toLowerCase()); + } + }); + } catch (err) {} +} + +async function find_symbols(req, all_words) { + try { + req.body.string.match(/[ \[\]:"\\|,.<>\/?~`!@#$%^&*()_+\-={};']/gi).forEach((item) => { + if (item !== " " && !all_words.symbol.includes(item)) { + all_words.symbol.push(item); + } + }); + } catch (err) {} +} + +async function find_numbers(req, all_words) { + try { + req.body.string.match(/(\d+)/g).forEach((item) => { + if (!all_words.number.includes(item)) { + all_words.number.push(item); + } + }); + } catch (err) {} +} + +async function convert_numbers(req) { + try { + numbers_to_letters = { + "4": "a", + "8": "b", + "3": "e", + "1": "l", + "0": "o", + "5": "s", + "7": "t", + "2": "z" + } + + temp_value = "" + for (i = 0; i < req.body.string.length; i++) { + _temp = numbers_to_letters[req.body.string.charAt(i)] + if (_temp != undefined) { + temp_value += numbers_to_letters[req.body.string.charAt(i)]; + } else { + temp_value += req.body.string.charAt(i); + } + } + req.body.string = temp_value + } catch (err) {} +} + +async function get_maybe_words(req, all_words) { + await WordsNinja.loadDictionary(); + all_words.maybe = await WordsNinja.splitSentence(req.body.string).filter(function(elem, index, self) { + return index === self.indexOf(elem); + }).filter(word => word.length > 1); +} + +module.exports = { + get_maybe_words, + find_symbols, + find_numbers, + convert_numbers, + split_upper_case, + split_alphabet_case, + most_common, + find_other, + analyze_string +} diff --git a/package.json b/package.json index f2bb610..e515bb4 100644 --- a/package.json +++ b/package.json @@ -1,14 +1,15 @@ { "name": "social-analyzer", - "version": "2.0.2", + "version": "2.0.10", "description": "Advanced Social Media Profiles Finder and String Analysis Tool", - "main": "server.js", + "main": "app.js", "scripts": { "test": "echo \"Error: no test specified\" && exit 1", "start": "node app.js" }, "dependencies": { "async": "3.2.0", + "cheerio": "1.0.0-rc.5", "express": "^4.16.3", "express-async-errors": "3.1.1", "follow-redirects": "1.13.0", @@ -28,8 +29,7 @@ "tmp": "0.2.1", "wink-tokenizer": "1.1.0", "wordsninja": "1.0.0", - "yargs": "16.2.0", - "cheerio": "1.0.0-rc.5" + "yargs": "16.2.0" }, "author": "QeeqBox", "license": "AGPL-3.0", diff --git a/public/app.html b/public/app.html index 297d252..eb3ca68 100644 --- a/public/app.html +++ b/public/app.html @@ -1,6 +1,6 @@