From 3d21a7658a4bd542e57584645b5f2bbeeb36dd3f Mon Sep 17 00:00:00 2001
From: G <52905881+giga-a@users.noreply.github.com>
Date: Wed, 30 Dec 2020 14:33:44 -0800
Subject: [PATCH] =?UTF-8?q?2020.V.01.16=20added=20CLI=20=F0=9F=91=8F?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
README.md | 19 +-
app.js | 1134 ++++++++++++++++++++++++++++++++++++++++++++++++++
info | 4 +-
package.json | 5 +-
sites.json | 8 +-
5 files changed, 1160 insertions(+), 10 deletions(-)
create mode 100644 app.js
diff --git a/README.md b/README.md
index 59ad123..23f33cf 100644
--- a/README.md
+++ b/README.md
@@ -10,6 +10,7 @@ I have been getting requests from all over the place asking me to add a python C
**Please submit your contribution in a Pull Request!**
## Updates
+- (New) Added a small CLI (You can use this project from Command Line) đź‘Ź
- Added an optional timeout and implicit wait for each detection (Some websites have a delay logic implemented in the backend)
- Added logs (user request)
- Added 5 seconds timeout to https.get (user request)
@@ -50,7 +51,7 @@ Profile images **will not** be blurred. If you want them to be blurred, turn tha
## Special Detections
- facebook
-## Install and run (Nodejs + NPM + Firefox)
+## Install and run as web app (Nodejs + NPM + Firefox)
```bash
sudo add-apt-repository ppa:mozillateam/ppa
sudo apt-get update
@@ -63,7 +64,21 @@ npm install
npm start
```
-## Install and run (docker)
+## Install and run as cli (Nodejs + NPM + Firefox)
+```bash
+sudo add-apt-repository ppa:mozillateam/ppa
+sudo apt-get update
+sudo apt-get install -y firefox-esr tesseract-ocr git
+git clone https://github.com/qeeqbox/social-analyzer.git
+cd social-analyzer
+rm -rf package-lock.json node_modules
+npm install lodash
+npm install
+# if you want to list all websites use node app.js -c -l
+node app.js -c -u "username" -w "youtube pinterest tumblr"
+```
+
+## Install and run as web app (docker)
```bash
git clone https://github.com/qeeqbox/social-analyzer.git
cd social-analyzer
diff --git a/app.js b/app.js
new file mode 100644
index 0000000..6c4d83b
--- /dev/null
+++ b/app.js
@@ -0,0 +1,1134 @@
+var google_api_key = "";
+var google_api_cs = "";
+
+console.log('[!] Detections are updated every often, make sure to get the most updated ones');
+
+var semver = require('semver');
+if (semver.satisfies(process.version, '>13 || <13')) {
+ console.log('[Good] NodeJS Version Check');
+} else {
+ console.log('[Error] NodeJS Version Check');
+ process.exit(1);
+}
+
+var argv = require('yargs')
+ .alias('c', 'cli')
+ .alias('u', 'user')
+ .alias('w', 'website')
+ .alias('o', 'output')
+ .alias('l', 'list')
+ .usage('Usage: $0 -s "user" -w "website" -o "output"')
+ .example('$0 -s "joe" -w "facebook"', 'check user joe in facebook website')
+ .example('$0 -s "natalie" -w "facebook wordpress"', 'check if user natalie in facebook & wordpress website')
+ .describe('u', 'a user or stirng')
+ .describe('w', 'a website or websites sparated with space')
+ .describe('o', 'option output file')
+ .describe('l', 'list all available websites')
+ .help('h')
+ .alias('h', 'help')
+ .argv;
+
+var tmp = require("tmp");
+var express = require("express");
+var fs = require("fs");
+var tokenizer = require("wink-tokenizer");
+var axios = require("axios")
+var WordsNinjaPack = require("wordsninja");
+var generatorics = require("generatorics");
+var {
+ findWord
+} = require("most-common-words-by-language");
+var tesseract = require("node-tesseract-ocr");
+var url = require("url");
+var sanitizeHtml = require("sanitize-html");
+var firefox = require("selenium-webdriver/firefox");
+var {
+ Builder,
+ By
+} = require("selenium-webdriver");
+var util = require('util');
+var https = require("follow-redirects").https;
+var async = require("async");
+var PrettyError = require('pretty-error');
+var pe = new PrettyError();
+require('express-async-errors');
+//var jsdom = require('jsdom');
+//var dom = new jsdom.JSDOM();
+//var window = dom.window;
+//var document = window.document;
+//var $ = require('jquery')(window);
+const {
+ htmlToText
+} = require('html-to-text');
+var _tokenizer = tokenizer();
+var parsed_json = JSON.parse(fs.readFileSync("dict.json"));
+var parsed_sites = JSON.parse(fs.readFileSync("sites.json"));
+var app = express();
+
+var WordsNinja = new WordsNinjaPack();
+app.use(express.urlencoded({
+ extended: true
+}));
+app.use(express.json());
+app.use(express.static("public"));
+
+if (!fs.existsSync('logs')) {
+ fs.mkdirSync('logs');
+}
+
+var logs_queue = Promise.resolve();
+
+function log_to_file_queue(uuid, msg) {
+ logs_queue = logs_queue.then(function() {
+ return new Promise(function(resolve) {
+ fs.appendFile("logs/" + uuid + "_log.txt", msg + "\n", function(err, data) {
+ console.log(msg)
+ resolve();
+ });
+ });
+ });
+}
+
+app.post("/get_logs", async function(req, res, next) {
+ var last_line = "nothinghere"
+ if (req.body.uuid != "") {
+ req.body.uuid = req.body.uuid.replace(/[^a-zA-Z0-9\-]+/g, '');
+ var data = fs.readFileSync("logs/" + req.body.uuid + "_log.txt").toString();
+ if (typeof data !== 'undefined' && data) {
+ last_line = data.split('\n').slice(-2)[0];
+ }
+ res.send(last_line)
+ }
+})
+
+function get_site_from_url(_url) {
+ temp = url.parse(_url.replace("{username}", "nothinghere")).hostname
+ return temp.replace("nothinghere.", "")
+}
+
+async function find_username_special(req) {
+ const time = new Date();
+ const functions = [];
+ parsed_sites.forEach((site) => {
+ if ("status" in site) {
+ if (site.status == "bad") {
+ return Promise.resolve();
+ }
+ }
+ if ("name" in site) {
+ if (site.name == "facebook") {
+ if (site.selected == "true") {
+ functions.push(find_username_site_special_facebook_1.bind(null, req.body.uuid, req.body.string, site));
+ }
+ }
+ }
+ });
+ const results = await async.parallelLimit(functions, 5);
+ console.log(`Total time ${new Date() - time}`);
+ return results.filter(item => item !== undefined)
+}
+
+async function find_username_site_special_facebook_1(uuid, username, site) {
+ return new Promise(async (resolve, reject) => {
+ log_to_file_queue(uuid, "[Checking] " + get_site_from_url(site.url))
+ let driver = new Builder()
+ .forBrowser("firefox")
+ .setFirefoxOptions(new firefox.Options().headless().windowSize({
+ width: 640,
+ height: 480
+ }))
+ .build();
+
+ try {
+ var timeouts = {
+ implicit: 0,
+ pageLoad: 10000,
+ script: 10000
+ };
+
+ var source = "";
+ var data = "";
+ var text_only = "unavailable";
+ var title = "unavailable";
+ var temp_profile = {
+ "found": 0,
+ "image": "",
+ "link": "",
+ rate: "",
+ title: "",
+ text: ""
+ };
+ var link = "https://mbasic.facebook.com/login/identify/?ctx=recoveqr";
+ await driver.manage().setTimeouts(timeouts);
+ await driver.get(link);;
+ await driver.findElement(By.id('identify_search_text_input')).sendKeys(username);
+ await driver.findElement(By.id('did_submit')).click();
+ source = await driver.getPageSource();
+ text_only = await driver.findElement(By.tagName("body")).getText();
+ await driver.quit()
+ if (source.includes("Try Entering Your Password")) {
+ temp_found = "true";
+ temp_profile.found += 1
+ }
+ if (temp_profile.found > 0) {
+ temp_profile.text = "unavailable";
+ temp_profile.title = "unavailable";
+ temp_profile.rate = "%" + ((temp_profile.found / 1) * 100).toFixed(2);
+ temp_profile.link = site.url.replace("{username}", username);
+ resolve(temp_profile);
+ } else {
+ resolve(undefined)
+ }
+ } catch (err) {
+ if (driver !== undefined) {
+ try {
+ await driver.quit()
+ } catch (err) {
+ console.log("Driver Session Issue")
+ }
+ }
+ resolve(undefined)
+ }
+ });
+}
+
+async function find_username_advanced(req) {
+ const time = new Date();
+ const functions = [];
+ parsed_sites.forEach((site) => {
+ if ("status" in site) {
+ if (site.status == "bad") {
+ return Promise.resolve();
+ }
+ }
+ if (site.selected == "true" && site.detections.length > 0) {
+ functions.push(find_username_site_new.bind(null, req.body.uuid, req.body.string, req.body.option, site));
+ }
+ });
+ const results = await async.parallelLimit(functions, 5);
+ console.log(`Total time ${new Date() - time}`);
+ return results.filter(item => item !== undefined)
+}
+
+async function find_username_site_new(uuid, username, options, site) {
+ return new Promise(async (resolve, reject) => {
+ log_to_file_queue(uuid, "[Checking] " + get_site_from_url(site.url))
+ let driver = new Builder()
+ .forBrowser("firefox")
+ .setFirefoxOptions(new firefox.Options().headless().windowSize({
+ width: 640,
+ height: 480
+ }))
+ .build();
+
+ try {
+
+ var timeouts = {
+ implicit: 0,
+ pageLoad: 5000,
+ script: 5000
+ };
+
+ var timeout = (site.timeout != 0) ? site.timeout * 1000 : 5000;
+ var implicit = (site.implicit != 0) ? site.implicit * 1000 : 0;
+
+ timeouts = {
+ implicit: implicit,
+ pageLoad: timeout,
+ script: timeout
+ };
+
+ console.log(timeouts)
+
+ var source = "";
+ var data = "";
+ var text_only = "unavailable";
+ var title = "unavailable";
+ var temp_profile = {
+ "found": 0,
+ "image": "",
+ "link": "",
+ rate: "",
+ title: "",
+ text: ""
+ };
+ var link = site.url.replace("{username}", username);
+ await driver.manage().setTimeouts(timeouts);
+ await driver.get(link);;
+ source = await driver.getPageSource();
+ data = await driver.takeScreenshot();
+ title = await driver.getTitle();
+ text_only = await driver.findElement(By.tagName("body")).getText();
+ await driver.quit()
+ if (options.includes("ShowUserProflesSlow")) {
+ temp_profile["image"] = "data:image/png;base64,{image}".replace("{image}", data);
+ }
+ if (site.selected == "true" && site.detections.length > 0 && options.includes("FindUserProflesSlow")) {
+ await Promise.all(site.detections.map(async detection => {
+ try {
+ if ("status" in detection) {
+ if (detection.status == "bad") {
+ return;
+ }
+ }
+ var temp_found = "false"
+ if (detection.type == "ocr" && data != "") {
+ tmpobj = tmp.fileSync();
+ fs.writeFileSync(tmpobj.name, Buffer.from(data, "base64"));
+ await tesseract.recognize(tmpobj.name, {
+ lang: "eng",
+ oem: 1,
+ psm: 3,
+ })
+ .then(text => {
+ text = text.replace(/[^A-Za-z0-9]/gi, "");
+ detection.string = detection.string.replace(/[^A-Za-z0-9]/gi, "");
+ if (text != "") {
+ if (text.toLowerCase().includes(detection.string.toLowerCase())) {
+ temp_found = "true";
+ }
+ if (detection.return == temp_found) {
+ //console.log(text);
+ //console.log(detection.string," > Found ocr");
+ temp_profile.found += 1;
+ }
+ }
+ })
+ .catch(error => {
+ console.log(error.message);
+ })
+ tmpobj.removeCallback();
+ } else if (detection.type == "normal" && source != "") {
+ if (source.toLowerCase().includes(detection.string.replace("{username}", username).toLowerCase())) {
+ temp_found = "true";
+ }
+ if (detection.return == temp_found) {
+ //console.log(detection.string," > normal");
+ temp_profile.found += 1
+ }
+ }
+
+ } catch (err) {
+
+ }
+ }));
+ }
+ if (temp_profile.found > 0 || temp_profile.image != "") {
+ temp_profile.text = sanitizeHtml(text_only);
+ temp_profile.title = sanitizeHtml(title);
+ temp_profile.rate = "%" + ((temp_profile.found / site.detections.length) * 100).toFixed(2);
+ temp_profile.link = site.url.replace("{username}", username);
+ resolve(temp_profile);
+ } else {
+ resolve(undefined)
+ }
+ } catch (err) {
+ if (driver !== undefined) {
+ try {
+ await driver.quit()
+ } catch (err) {
+ console.log("Driver Session Issue")
+ }
+ }
+ resolve(undefined)
+ }
+ });
+}
+
+async function find_username_normal(req) {
+
+ var functions = [];
+ var detections_result = [];
+
+ async function find_username_site(uuid, username, options, site, body) {
+ try {
+ log_to_file_queue(uuid, "[Checking] " + get_site_from_url(site.url))
+ var detections_count = 0;
+ var source = body;
+ var text_only = "unavailable";
+ var title = "unavailable";
+ var temp_profile = {
+ "found": 0,
+ "image": "",
+ "link": "",
+ rate: "",
+ title: "",
+ text: ""
+ };
+ await Promise.all(site.detections.map(async detection => {
+ var temp_found = "false";
+ if (detection.type == "normal" && options.includes("FindUserProflesFast") && source != "" && detection.return == "true") {
+ detections_count += 1
+ if (source.toLowerCase().includes(detection.string.replace("{username}", username).toLowerCase())) {
+ temp_found = "true";
+ }
+ if (detection.return == temp_found) {
+ //console.log(detection.string, " > normal");
+ temp_profile.found += 1
+ }
+ }
+ }));
+ if (temp_profile.found > 0 && detections_count != 0) {
+ temp_profile.text = htmlToText(body, {
+ wordwrap: false,
+ hideLinkHrefIfSameAsText: true,
+ ignoreHref: true,
+ ignoreImage: true
+ });
+ if (temp_profile.text == "") {
+ temp_profile.text = "unavailable"
+ }
+ temp_profile.title = sanitizeHtml(title);
+ temp_profile.rate = "%" + ((temp_profile["found"] / detections_count) * 100).toFixed(2);
+ temp_profile.link = site.url.replace("{username}", username);
+ return Promise.resolve(temp_profile);
+ }
+ return Promise.resolve();
+ } catch (err) {
+ return Promise.reject();
+ }
+ }
+
+ async function find_username_sites(uuid, username, options, parsed_sites) {
+
+ await parsed_sites.forEach(site => {
+ if ("status" in site) {
+ if (site.status == "bad") {
+ return;
+ }
+ }
+ if (site.selected == "true" && site.detections.length > 0) {
+ functions.push(function(callback) {
+ var request = https.get(site.url.replace("{username}", username), function(res) {
+ var body = ""
+ res.on("data", function(chunk) {
+ body += chunk;
+ });
+ res.on("end", async function() {
+ var results = await find_username_site(uuid, username, options, site, body);
+ detections_result.push(results);
+ callback(null, "Done!");
+ });
+ });
+ request.on('error', function(e) {
+ callback(null, "Done!");
+ });
+ request.on('socket', function(socket) {
+ var timeout = (site.timeout != 0) ? site.timeout * 1000 : 5000;
+ socket.setTimeout(timeout, function() {
+ request.abort();
+ });
+ });
+ });
+ }
+ });
+ }
+
+ await find_username_sites(req.body.uuid, req.body.string, req.body.option, parsed_sites);
+ await async.parallelLimit(functions, 100);
+ return detections_result.filter(item => item !== undefined);
+}
+
+async function find_username_advanced_2(username, options) {
+
+ var detections_result = [];
+
+ async function find_username_site(username, options, driver, site) {
+ try {
+ if ("status" in site) {
+ if (site.status == "bad") {
+ return Promise.resolve();
+ }
+ }
+ if (site.selected == "true" && site.detections.length > 0 || site.selected == "true" && options.includes("ShowUserProflesSlow")) {
+ var source = "";
+ var data = "";
+ var text_only = "unavailable";
+ var title = "unavailable";
+ var temp_profile = {
+ "found": 0,
+ "image": "",
+ "link": "",
+ rate: "",
+ title: "",
+ text: ""
+ };
+ var link = site.url.replace("{username}", username);
+ await driver.get(link);;
+ source = await driver.getPageSource();
+ data = await driver.takeScreenshot();
+ title = await driver.getTitle();
+ text_only = await driver.findElement(By.tagName("body")).getText();
+ if (options.includes("ShowUserProflesSlow")) {
+ temp_profile["image"] = "data:image/png;base64,{image}".replace("{image}", data);
+ }
+ if (site.selected == "true" && site.detections.length > 0 && options.includes("FindUserProflesSlow")) {
+ await Promise.all(site.detections.map(async detection => {
+ if ("status" in detection) {
+ if (detection.status == "bad") {
+ return;
+ }
+ }
+ var temp_found = "false"
+ if (detection.type == "ocr" && data != "") {
+ tmpobj = tmp.fileSync();
+ fs.writeFileSync(tmpobj.name, Buffer.from(data, "base64"));
+ await tesseract.recognize(tmpobj.name, {
+ lang: "eng",
+ oem: 1,
+ psm: 3,
+ })
+ .then(text => {
+ text = text.replace(/[^A-Za-z0-9]/gi, "");
+ detection.string = detection.string.replace(/[^A-Za-z0-9]/gi, "");
+ if (text != "") {
+ if (text.toLowerCase().includes(detection.string.toLowerCase())) {
+ temp_found = "true";
+ }
+ if (detection.return == temp_found) {
+ //console.log(text);
+ //console.log(detection.string," > Found ocr");
+ temp_profile.found += 1;
+ }
+ }
+ })
+ .catch(error => {
+ console.log(error.message);
+ })
+ tmpobj.removeCallback();
+ } else if (detection.type == "normal" && source != "") {
+ if (source.toLowerCase().includes(detection.string.replace("{username}", username).toLowerCase())) {
+ temp_found = "true";
+ }
+ if (detection.return == temp_found) {
+ //console.log(detection.string," > normal");
+ temp_profile.found += 1
+ }
+ }
+ }));
+ }
+ if (temp_profile.found > 0 || temp_profile.image != "") {
+ temp_profile.text = sanitizeHtml(text_only);
+ temp_profile.title = sanitizeHtml(title);
+ temp_profile.rate = "%" + ((temp_profile.found / site.detections.length) * 100).toFixed(2);
+ temp_profile.link = site.url.replace("{username}", username);
+ return Promise.resolve(temp_profile);
+ }
+ }
+ return Promise.resolve();
+ } catch (err) {
+ return Promise.reject();
+ }
+ }
+
+ async function find_username_sites(username, options, driver, parsed_sites) {
+ for (var site of parsed_sites) {
+ var result = await find_username_site(username, options, driver, site);
+ detections_result.push(result);
+ }
+
+ return detections_result;
+ }
+
+ let driver = new Builder()
+ .forBrowser("firefox")
+ .setFirefoxOptions(new firefox.Options().headless().windowSize({
+ width: 640,
+ height: 480
+ }))
+ .build();
+
+ var timeouts = {
+ implicit: 0,
+ pageLoad: 10000,
+ script: 10000
+ };
+
+ await driver.manage().setTimeouts(timeouts);
+ var results = await find_username_sites(username, options, driver, parsed_sites);
+ await driver.quit();
+ return results.filter(item => item !== undefined);
+}
+
+app.get("/get_settings", async function(req, res, next) {
+ temp_list = [];
+ temp_list = await Promise.all(parsed_sites.map(async (site, index) => {
+ var temp_url = "";
+ if ("status" in site) {
+ if (site.status == "bad") {
+ return Promise.resolve();
+ }
+ }
+ if (site.detections.length > 0) {
+ temp_url = url.parse(site.url.replace("{username}", "nothinghere")).hostname
+ temp_url = temp_url.replace("nothinghere.", "");
+ if (temp_url != "nothinghere") {
+ temp_selected = "false";
+ if ("selected" in site) {
+ if (site.selected == "true") {
+ temp_selected = "true";
+ }
+ }
+ return Promise.resolve({
+ "index": index,
+ "url": temp_url,
+ "selected": temp_selected
+ });
+ }
+ }
+
+ return Promise.resolve();
+ }));
+
+ temp_list = temp_list.filter(item => item !== undefined);
+ temp_list.sort(function(a, b) {
+ var keyA = a.url,
+ keyB = b.url;
+ // Compare the 2 dates
+ if (keyA < keyB) return -1;
+ if (keyA > keyB) return 1;
+ return 0;
+ });
+ res.json({
+ google: [google_api_key.substring(0, 10) + "******", google_api_cs.substring(0, 10) + "******"],
+ detections: temp_list
+ });
+});
+
+app.post("/save_settings", async function(req, res, next) {
+ await parsed_sites.forEach(function(value, i) {
+ parsed_sites[i].selected = "false"
+ });
+ if ("detections" in req.body) {
+ if (req.body.detections.length > 0) {
+ await req.body.detections.split(',').forEach(item => {
+ parsed_sites[Number(item)].selected = "true";
+ });
+ }
+ }
+ if (req.body.google_key != google_api_key.substring(0, 10) + "******") {
+ google_api_key = req.body.google_key;
+ }
+ if (req.body.google_cv != google_api_cs.substring(0, 10) + "******") {
+ google_api_cs = req.body.google_cv;
+ }
+
+ res.json("Done");
+});
+
+app.get("/generate", async function(req, res, next) {
+ var list_of_combinations = []
+ if (req.body.option == "Generate") {
+ if (req.body.words != undefined && req.body.words.length > 1 && req.body.words.length < 8) {
+ for (var perm of generatorics.permutationCombination(req.body.words)) {
+ if (perm.join("") !== "") {
+ list_of_combinations.push(perm.join(""));
+ }
+ }
+ }
+ }
+ res.json({
+ combinations: list_of_combinations
+ });
+
+});
+
+async function get_words_info(all_words, words_info) {
+ var temp_added = []
+ for (let all_words_key of Object.keys(all_words)) {
+ for (let all_words_word of all_words[all_words_key]) {
+ if (!temp_added.includes(all_words_word)) {
+ temp_added.push(all_words_word);
+ var temp_words_info = {
+ "word": all_words_word,
+ "text": "",
+ "results": []
+ }
+ try {
+ var url1 = "https://api.duckduckgo.com/?q={0}&format=json&pretty=1&no_html=1&skip_disambig=1".replace("{0}", all_words_word);
+ var url2 = "https://api.duckduckgo.com/?q={0}&format=json&pretty=1".replace("{0}", all_words_word);
+ var response1 = await axios.get(url1);
+ var response2 = await axios.get(url2);
+ if (response2.status === 200) {
+ if ("RelatedTopics" in response2.data) {
+ if (response2.data.RelatedTopics.length > 0) {
+ if (response1.status === 200) {
+ if ("AbstractText" in response1.data && response1.data.AbstractText != "") {
+ temp_words_info.text = response1.data.AbstractText;
+ } else if ("Abstract" in response1.data && response1.data.AbstractText != "") {
+ temp_words_info.text = response1.data.Abstract;
+ } else {
+ temp_words_info.text = "unknown";
+ }
+ }
+ response2.data.RelatedTopics.forEach(function(item) {
+ if ("Name" in item) {
+ item.Topics.forEach(function(topic) {
+ temp_words_info.results.push({
+ "type": item.Name,
+ "text": topic.Text,
+ "url": topic.FirstURL
+ });
+ });
+ } else {
+ temp_words_info.results.push({
+ "type": "Related",
+ "text": item.Text,
+ "url": item.FirstURL
+ });
+ }
+ });
+ }
+ }
+ }
+
+ if (temp_words_info.results.length > 0) {
+ words_info.push(temp_words_info);
+ }
+ } catch (error) {
+ console.log(error);
+ }
+ }
+ }
+ }
+}
+
+async function check_engines(req, info) {
+ try {
+ if (google_api_key == "" || google_api_cs == "") {
+ return
+ }
+ var url = "https://www.googleapis.com/customsearch/v1?key={0}&cx={1}&q={2}".replace("{0}", google_api_key).replace("{1}", google_api_cs).replace("{2}", req.body.string);
+ var response = await axios.get(url);
+ if (response.status === 200) {
+ try {
+ info.original = response.data.queries.request[0].searchTerms
+ } catch (e) {}
+ try {
+ info.corrected = response.data.spelling.correctedQuery
+ } catch (e) {}
+ try {
+ info.total = response.data.searchInformation.totalResults
+ } catch (e) {}
+ try {
+ response.data.items.forEach(function(item) {
+ info["items"].push({
+ "title": item.title,
+ "snippet": item.snippet
+ });
+ });
+ } catch (e) {}
+ try {
+ if (info.total == 0 && info.corrected != "") {
+ info.checking = info.original + " [Error]
Try this: " + info.corrected;
+ } else if (info.total > 0 && info.corrected != "") {
+ info.checking = info.original + " [Good]
Suggested word: " + info.corrected + "
Total lookups: " + info.total;
+ } else if (info.total > 0 && info.corrected == "") {
+ info.checking = info.original + " [Good]
Total lookups: " + info.total;
+ } else {
+ info.checking = "Using " + info.original + " with no lookups";
+ }
+ } catch (e) {}
+ }
+ } catch (error) {
+ console.log(error);
+ }
+}
+
+function most_common1(all_words, temp_words) {
+ var temp_list = []
+ Object.keys(all_words).forEach(function(key) {
+ all_words[key].forEach(function(item) {
+ if (!temp_list.includes(item) && item.length > 1) {
+ temp_list.push(item);
+ var temp = findWord(item);
+ if (Object.keys(temp).length != 0) {
+ var languages = Object.keys(temp).map(function(key) {
+ return [key, temp[key]];
+ });
+ languages.sort(function(first, second) {
+ return second[1] - first[1]
+ }).reverse();
+ temp_words.push({
+ "word": item,
+ "languages": languages.map(e => e.join(":")).join(" ")
+ });
+ }
+ }
+ });
+ });
+}
+
+async function most_common(all_words, temp_words) {
+ var temp_list = []
+ Object.keys(all_words).forEach(function(key) {
+ all_words[key].forEach(function(item) {
+ if (!temp_list.includes(item) && item.length > 1) {
+ temp_list.push(item);
+ var temp = findWord(item);
+ if (Object.keys(temp).length != 0) {
+ var languages = Object.keys(temp).map(function(key) {
+ return [key, temp[key]];
+ });
+ languages.sort(function(first, second) {
+ return second[1] - first[1]
+ }).reverse();
+ temp_words.push({
+ "word": item,
+ "languages": languages.map(e => e[0]).join(", ")
+ });
+ }
+ }
+ });
+ });
+}
+
+function find_other(all_words) {
+ var words = WordsNinja.splitSentence(req.body.string);
+
+ words.forEach(function(word) {
+ var value = false
+ Object.keys(all_words).forEach(function(key) {
+ if (all_words[key].includes(word)) {
+ value = true
+ }
+ });
+
+ if (!value && !all_words.maybe.includes(word)) {
+ all_words.maybe.push(word);
+ }
+ });
+}
+
+function remove_word(str, sub_string) {
+ part1 = str.substring(0, str.indexOf(sub_string));
+ part2 = str.substring(str.indexOf(sub_string) + sub_string.length, str.length);
+ temp = (part1 + part2).replace(/[ \[\]:"\\|,.<>\/?~`!@#$%^&*()_+\-={};"]/gi, "");
+ return temp;
+}
+
+async function analyze_name(req, all_words) {
+ log_to_file_queue(req.body.uuid, "[Starting] String analysis")
+ temp_rr_names = []
+ string_to_check = req.body.string
+ parsed_json.prefix.forEach(function(item, index) {
+ if (string_to_check.indexOf(item) == 0 && !all_words.prefix.includes(item)) {
+ all_words.prefix.push(item);
+ temp = remove_word(string_to_check, item);
+ if (temp !== null && temp !== "" && !all_words.unknown.includes(temp) && !all_words.maybe.includes(temp) && temp.length > 1) {
+ all_words.unknown.push(temp);
+ }
+ }
+ });
+ parsed_json.m_names.forEach(function(item, index) {
+ if (string_to_check.indexOf(item) >= 0 && !all_words.name.includes(item)) {
+ all_words.name.push(item);
+ temp = remove_word(string_to_check, item);
+ if (temp !== null && temp !== "" && !all_words.unknown.includes(temp) && !all_words.maybe.includes(temp) && temp.length > 1) {
+ all_words.unknown.push(temp);
+ }
+ }
+ });
+ parsed_json.f_names.forEach(function(item, index) {
+ if (string_to_check.indexOf(item) >= 0 && !all_words.name.includes(item)) {
+ all_words.name.push(item);
+ temp = remove_word(string_to_check, item);
+ if (temp !== null && temp !== "" && !all_words.unknown.includes(temp) && !all_words.maybe.includes(temp) && temp.length > 1) {
+ all_words.unknown.push(temp);
+ }
+ }
+ });
+
+ all_words.prefix.forEach(function(h_item, index) {
+ all_words.unknown.forEach(function(r_item, index) {
+ if (r_item.indexOf(h_item) == 0) {
+ temp = remove_word(r_item, h_item);
+ if (temp !== null && temp !== "" && !temp_rr_names.includes(temp) && !all_words.maybe.includes(temp) && temp.length > 1) {
+ temp_rr_names.push(temp);
+ }
+ }
+ });
+ });
+
+ var temp_r_concat = all_words.unknown.concat(temp_rr_names.filter((item) => all_words.unknown.indexOf(item) < 0));
+
+ all_words.unknown = temp_r_concat
+ temp_rr_names = []
+
+ all_words.number.forEach(function(n_item, index) {
+ all_words.unknown.forEach(function(r_item, index) {
+ if (r_item.indexOf(n_item) >= 0) {
+ temp = remove_word(r_item, n_item);
+ if (temp !== null && temp !== "" && !temp_rr_names.includes(temp) && !all_words.maybe.includes(temp) && temp.length > 1) {
+ temp_rr_names.push(temp);
+ }
+ }
+ });
+ });
+
+ var temp_r_concat = all_words.unknown.concat(temp_rr_names.filter((item) => all_words.unknown.indexOf(item) < 0));
+ all_words.unknown = temp_r_concat
+ log_to_file_queue(req.body.uuid, "[Done] String analysis")
+}
+
+app.post("/url", async function(req, res, next) {
+ await WordsNinja.loadDictionary();
+ var info = {
+ "items": [],
+ "original": "",
+ "corrected": "",
+ "total": 0,
+ "checking": "Using " + req.body.string + " with no lookups"
+ }
+ var user_info_normal = {
+ data: {},
+ type: "all"
+ }
+ var user_info_advanced = {
+ data: {},
+ type: "all"
+ }
+ var user_info_special = {
+ data: {},
+ type: "all"
+ }
+ var all_words = {
+ "prefix": [],
+ "name": [],
+ "number": [],
+ "symbol": [],
+ "unknown": [],
+ "maybe": []
+ }
+ var words_info = []
+ var temp_words = []
+ if (req.body.string == null || req.body.string == "") {
+ res.json("Error");
+ } else {
+ req.body.uuid = req.body.uuid.replace(/[^a-zA-Z0-9\-]+/g, '');
+ if (req.body.option.includes("FindUserProflesSpecial")) {
+ log_to_file_queue(req.body.uuid, "[Starting] Checking user profiles special")
+ user_info_special.data = await find_username_special(req);
+ log_to_file_queue(req.body.uuid, "[Done] Checking user profiles special")
+ }
+ if (req.body.option.includes("FindUserProflesFast")) {
+ log_to_file_queue(req.body.uuid, "[Starting] Checking user profiles normal")
+ user_info_advanced.data = await find_username_normal(req);
+ log_to_file_queue(req.body.uuid, "[Done] Checking user profiles normal")
+ }
+ if (req.body.option.includes("FindUserProflesSlow") || req.body.option.includes("ShowUserProflesSlow")) {
+ if (!req.body.option.includes("FindUserProflesSlow")) {
+ user_info_normal.type = "show"
+ } else if (!req.body.option.includes("ShowUserProflesSlow")) {
+ user_info_normal.type = "noshow"
+ }
+ log_to_file_queue(req.body.uuid, "[Starting] Checking user profiles advanced")
+ user_info_normal.data = await find_username_advanced(req);
+ log_to_file_queue(req.body.uuid, "[Done] Checking user profiles advanced")
+ }
+ if (req.body.option.includes("LookUps")) {
+ log_to_file_queue(req.body.uuid, "[Starting] Lookup")
+ await check_engines(req, info);
+ log_to_file_queue(req.body.uuid, "[Done] Lookup")
+ }
+ if (req.body.option.includes("SplitWordsByUpperCase")) {
+ try {
+ req.body.string.match(/[A-Z][a-z]+/g).forEach((item) => {
+ if (item.length > 1 && !all_words.unknown.includes(item) && !all_words.maybe.includes(item)) {
+ all_words.unknown.push(item.toLowerCase());
+ }
+ });
+ log_to_file_queue(req.body.uuid, "[Done] Split by UpperCase")
+ } catch (err) {}
+ }
+
+ if (req.body.option.includes("SplitWordsByAlphabet")) {
+ try {
+ req.body.string.match(/[A-Za-z]+/g).forEach((item) => {
+ if (item.length > 1 && !all_words.unknown.includes(item) && !all_words.maybe.includes(item)) {
+ all_words.unknown.push(item.toLowerCase());
+ }
+ });
+ log_to_file_queue(req.body.uuid, "[Done] Split by Alphabet")
+ } catch (err) {}
+ }
+
+ req.body.string = req.body.string.toLowerCase();
+
+ if (req.body.option.includes("ConvertNumbers")) {
+ numbers_to_letters = {
+ "4": "a",
+ "8": "b",
+ "3": "e",
+ "1": "l",
+ "0": "o",
+ "5": "s",
+ "7": "t",
+ "2": "z"
+ }
+
+ temp_value = ""
+ for (i = 0; i < req.body.string.length; i++) {
+ _temp = numbers_to_letters[req.body.string.charAt(i)]
+ if (_temp != undefined) {
+ temp_value += numbers_to_letters[req.body.string.charAt(i)];
+ } else {
+ temp_value += req.body.string.charAt(i);
+ }
+ }
+ req.body.string = temp_value
+ log_to_file_queue(req.body.uuid, "[Done] Convert numbers to letters")
+ }
+
+ if (req.body.option.includes("LookUps") ||
+ req.body.option.includes("WordInfo") ||
+ req.body.option.includes("MostCommon") ||
+ req.body.option.includes("SplitWordsByUpperCase") ||
+ req.body.option.includes("SplitWordsByAlphabet") ||
+ req.body.option.includes("FindSymbols") ||
+ req.body.option.includes("FindNumbers") ||
+ req.body.option.includes("ConvertNumbers")) {
+ if (req.body.option.includes("FindNumbers")) {
+ try {
+ req.body.string.match(/(\d+)/g).forEach((item) => {
+ if (!all_words.number.includes(item)) {
+ all_words.number.push(item);
+ }
+ });
+ } catch (err) {}
+ }
+
+ if (req.body.option.includes("FindSymbols")) {
+ try {
+ req.body.string.match(/[ \[\]:"\\|,.<>\/?~`!@#$%^&*()_+\-={};']/gi).forEach((item) => {
+ if (item !== " " && !all_words.symbol.includes(item)) {
+ all_words.symbol.push(item);
+ }
+ });
+ } catch (err) {}
+ }
+
+ if (req.body.option.includes("SplitUpperCase")) {
+ req.body.string = req.body.string.replace(/([A-Z]+)/g, " $1");
+ if (req.body.string.startsWith(" ")) {
+ req.body.string = req.body.string.substring(1);
+ }
+ }
+ all_words.maybe = WordsNinja.splitSentence(req.body.string).filter(function(elem, index, self) {
+ return index === self.indexOf(elem);
+ }).filter(word => word.length > 1);
+ await analyze_name(req, all_words);
+ //find_other(all_words)
+ Object.keys(all_words).forEach((key) => (all_words[key].length == 0) && delete all_words[key]);
+
+ if (req.body.option.includes("MostCommon")) {
+ await most_common(all_words, temp_words);
+ }
+ if (req.body.option.includes("WordInfo")) {
+ await get_words_info(all_words, words_info);
+ }
+ } else if (req.body.option.includes("NormalAnalysis@@")) {
+ var maybe_words = WordsNinja.splitSentence(req.body.string);
+ all_words.maybe = maybe_words.filter(function(elem, index, self) {
+ return index === self.indexOf(elem);
+ });
+ list_of_tokens = _tokenizer.tokenize(req.body.string);
+ list_of_tokens.forEach(function(item, index) {
+ if (item.tag in all_words) {
+ all_words[item.tag].push(item.token);
+ } else {
+ all_words[item.tag] = []
+ all_words[item.tag].push(item.token);
+ }
+ });
+
+ Object.keys(all_words).forEach((key) => (all_words[key].length == 0) && delete all_words[key]);
+ }
+ res.json({
+ info,
+ table: all_words,
+ common: temp_words,
+ words_info: words_info,
+ user_info_normal: user_info_normal,
+ user_info_advanced: user_info_advanced,
+ user_info_special: user_info_special
+ });
+ }
+});
+
+app.use((err, req, res, next) => {
+ console.log(" --- Global Error ---")
+ console.log(pe.render(err));
+ res.json("Error");
+});
+
+process.on('uncaughtException', function(err) {
+ console.log(" --- Uncaught Error ---")
+ console.log(pe.render(err));
+})
+
+
+process.on('unhandledRejection', function(err) {
+ console.log(" --- Uncaught Rejection ---")
+ console.log(pe.render(err));
+})
+
+const server_host = '0.0.0.0';
+const server_port = process.env.PORT || 9005;
+
+
+async function check_user_cli(username,websites) {
+ var ret = []
+ var random_string = Math.random().toString(36).substring(2);
+ var req = {'body':{'uuid':random_string, 'string':username, 'option':'FindUserProflesFast'}}
+ await parsed_sites.forEach(async function(value, i) {
+ parsed_sites[i].selected = "false"
+ if (websites.length > 0) {
+ await websites.split(' ').forEach(item => {
+ if (parsed_sites[i].url.toLowerCase().includes(item.toLowerCase())) {
+ parsed_sites[i].selected = "true"
+ }
+ });
+ }
+ });
+ ret = await find_username_normal(req)
+ if (typeof ret === 'undefined' || ret === undefined || ret.length == 0){
+ log_to_file_queue(req.body.uuid,'User does not exist (try FindUserProflesSlow or FindUserProflesSpecial)');
+ }
+ else{
+ await ret.forEach(item => {
+ delete item['title']
+ delete item['image']
+ delete item['text']
+ item['link'] = get_site_from_url(item['link'])
+ log_to_file_queue(req.body.uuid,item);
+ });
+ }
+ //console.log(util.inspect(ret, { colors: true }));
+};
+
+async function list_all_websites() {
+ var temp_arr = []
+ await parsed_sites.forEach(item => {
+ temp_arr.push(get_site_from_url(item.url))
+ });
+ console.log('[Listing] Available websites\n' + temp_arr.join('\n'))
+}
+
+if ('cli' in argv) {
+ if ('list' in argv)
+ {
+ list_all_websites();
+ }
+ else if ('user' in argv && 'website' in argv) {
+ if (argv.user != "" && argv.website != "") {
+ check_user_cli(argv.user,argv.website)
+ } else {
+ console.log("user or website is empty, use -h for help")
+ }
+ }
+} else {
+ var server = app.listen(server_port, server_host, function() {
+ console.log("Server started at http://%s:%s/app.html", server_host, server_port);
+ });
+}
diff --git a/info b/info
index c5bdfda..d36097d 100644
--- a/info
+++ b/info
@@ -1,8 +1,8 @@
-{"version":"2020.V.01.15",
+{"version":"2020.V.01.16",
"build":"pass",
"test":"pass",
"websites":"255",
"detections":"765",
"special":"1",
"awaiting_verification":"64",
- "auto_testing":"5ebdf895-46ae-4d37-89d8-da427b6609d1"}
+ "auto_testing":"ebe64ae3-3d15-46f7-82e3-57c047cf47d8"}
diff --git a/package.json b/package.json
index d75d5e9..cd6a04b 100644
--- a/package.json
+++ b/package.json
@@ -5,7 +5,7 @@
"main": "server.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1",
- "start": "node server.js"
+ "start": "node app.js"
},
"dependencies": {
"async": "3.2.0",
@@ -26,7 +26,8 @@
"semver": "7.3.4",
"tmp": "0.2.1",
"wink-tokenizer": "1.1.0",
- "wordsninja": "1.0.0"
+ "wordsninja": "1.0.0",
+ "yargs": "16.2.0"
},
"author": "QeeqBox",
"license": "AGPL 3.0"
diff --git a/sites.json b/sites.json
index 00879f2..972d255 100644
--- a/sites.json
+++ b/sites.json
@@ -1760,7 +1760,7 @@
"implicit": 0
},
{
- "url": "https://www.fl.ru/users/{username}/portfolio",
+ "url": "https://fl.ru/users/{username}/portfolio",
"detections": [{
"return": "false",
"string": "404 Not Found",
@@ -2082,7 +2082,7 @@
"implicit": 0
},
{
- "url": "https://www.fotolog.com/author/{username}",
+ "url": "https://fotolog.com/author/{username}",
"detections": [{
"return": "false",
"string": "That page doesn’t exist or is private",
@@ -2126,7 +2126,7 @@
"implicit": 0
},
{
- "url": "https://www.freelancer.com/u/{username}",
+ "url": "https://freelancer.com/u/{username}",
"detections": [{
"return": "false",
"string": "Looks like the page you are looking",
@@ -4389,7 +4389,7 @@
"implicit": 0
},
{
- "url": "https://www.quora.com/profile/{username}",
+ "url": "https://quora.com/profile/{username}",
"detections": [{
"return": "false",
"string": "Page Not Found",