From 8e6188906c435a9268015a758d4834ad462abdf1 Mon Sep 17 00:00:00 2001 From: G <52905881+giga-a@users.noreply.github.com> Date: Sat, 31 Jul 2021 03:14:47 -0700 Subject: [PATCH] =?UTF-8?q?[auto-b]=20multi-profile=20search=20?= =?UTF-8?q?=F0=9F=94=A5=F0=9F=94=A5=F0=9F=94=A5=20-=20ref=2042520c76ca92?= =?UTF-8?q?=20&=20361bd498e60e?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 3 +- app.js | 85 ++++++++++++++++++++++++++++---------- modules/fast-scan.js | 4 +- modules/helper.js | 1 + modules/slow-scan.js | 4 +- modules/string-analysis.js | 11 +++++ modules/visualize.js | 33 +++++++++++---- package-lock.json | 14 +++---- package.json | 2 +- public/app.html | 57 ++++++++++++------------- 10 files changed, 147 insertions(+), 67 deletions(-) diff --git a/README.md b/README.md index 58d3b01..4522aa6 100755 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ This project is *"currently used by some law enforcement agencies in countries w `Social Analyzer is in a league of its own and is a very impressive tool that I thoroughly recommend for Digital Investigators and OSINT practitioners` - by [Joseph Jones, Founder of Strategy Nord, Unita Insight and OS2INT](https://os2int.com/toolbox/investigating-usernames-with-social-analyzer/) ## Updates -- New search and stats options are queued for testing +- Multi-profile search is finally here! And, most of the public & private modules are compatible with this new search option - E.g. johndoe, janedoe <- any group of usernames separated by commas 🔥🔥🔥 ## So·cial Me·di·a Websites and applications that enable users to create and share content or to participate in social networking - Oxford Dictionary @@ -41,6 +41,7 @@ Standard localhost WEB APP url: http://0.0.0.0:9005/app.html ## Features - String & name analysis (Permutations and Combinations) - Find profile using multiple techniques (HTTPS library & Webdriver) +- Multi profile search (Used for correlation) - Multi layers detections (OCR, normal, advanced & special) - Visualized profile information using Ixora (Metadata & Patterns) - Metadata & Patterns extraction (Added from Qeeqbox osint project) diff --git a/app.js b/app.js index ee84a91..70fe18e 100755 --- a/app.js +++ b/app.js @@ -242,15 +242,15 @@ app.post("/analyze_string", async function(req, res, next) { "checking": "Using " + req.body.string + " with no lookups" } var user_info_normal = { - data: {}, + data: [], type: "all" } var user_info_advanced = { - data: {}, + data: [], type: "all" } var user_info_special = { - data: {}, + data: [], type: "all" } var all_words = { @@ -292,12 +292,34 @@ app.post("/analyze_string", async function(req, res, next) { username = req.body.string req.body.uuid = req.body.uuid.replace(/[^a-zA-Z0-9\-]+/g, ''); temp_uuid = req.body.uuid + helper.log_to_file_queue(req.body.uuid, "[Setting] Log file name: " + req.body.uuid) - helper.log_to_file_queue(req.body.uuid, "[Setting] Username: " + req.body.string) + + if (req.body.string.includes(",")){ + req.body.group = true + helper.log_to_file_queue(req.body.uuid, "[Setting] Multiple usernames: " + req.body.string) + } + else{ + helper.log_to_file_queue(req.body.uuid, "[Setting] Username: " + req.body.string) + } + if (req.body.option.includes("FindUserProfilesFast") || req.body.option.includes("GetUserProfilesFast")) { fast = true helper.log_to_file_queue(req.body.uuid, "[Starting] Checking user profiles normal") - user_info_normal.data = await fastScan.find_username_normal(req); + if(req.body.group){ + var old_string = req.body.string + const all_usernames = req.body.string.split(",").map( async item => { + req.body.string = item + var temp_arr = await fastScan.find_username_normal(req) + user_info_normal.data.push(...temp_arr) + }) + await Promise.all(all_usernames); + req.body.string = old_string + } + else{ + user_info_normal.data = await fastScan.find_username_normal(req); + } + helper.log_to_file_queue(req.body.uuid, "[Done] Checking user profiles normal") if (req.body.option.includes("CategoriesStats")) { helper.log_to_file_queue(req.body.uuid, "[Starting] Generate stats") @@ -334,25 +356,45 @@ app.post("/analyze_string", async function(req, res, next) { user_info_advanced.type = "noshow" } helper.log_to_file_queue(req.body.uuid, "[Starting] Checking user profiles advanced") - user_info_advanced.data = await slowScan.find_username_advanced(req); + + if(req.body.group){ + var old_string = req.body.string + const all_usernames = req.body.string.split(",").map( async item => { + req.body.string = item + var temp_arr = await slowScan.find_username_advanced(req); + user_info_advanced.data.push(...temp_arr) + }) + await Promise.all(all_usernames); + req.body.string = old_string + } + else{ + user_info_advanced.data = await slowScan.find_username_advanced(req); + } + helper.log_to_file_queue(req.body.uuid, "[Done] Checking user profiles advanced") } - if (req.body.option.includes("LookUps")) { - helper.log_to_file_queue(req.body.uuid, "[Starting] Lookup") - await externalApis.check_engines(req, info); - helper.log_to_file_queue(req.body.uuid, "[Done] Lookup") + if (!req.body.group){ + if (req.body.option.includes("LookUps")) { + helper.log_to_file_queue(req.body.uuid, "[Starting] Lookup") + await externalApis.check_engines(req, info); + helper.log_to_file_queue(req.body.uuid, "[Done] Lookup") + } + if (req.body.option.includes("CustomSearch")) { + helper.log_to_file_queue(req.body.uuid, "[Starting] Custom Search") + custom_search = await externalApis.custom_search_ouputs(req); + helper.log_to_file_queue(req.body.uuid, "[Done] Custom Search") + } + if (req.body.option.includes("FindOrigins")) { + helper.log_to_file_queue(req.body.uuid, "[Starting] Finding Origins") + names_origins = await nameAnalysis.find_origins(req); + helper.log_to_file_queue(req.body.uuid, "[Done] Finding Origins") + } } - if (req.body.option.includes("CustomSearch")) { - helper.log_to_file_queue(req.body.uuid, "[Starting] Custom Search") - custom_search = await externalApis.custom_search_ouputs(req); - helper.log_to_file_queue(req.body.uuid, "[Done] Custom Search") - } - if (req.body.option.includes("FindOrigins")) { - helper.log_to_file_queue(req.body.uuid, "[Starting] Finding Origins") - names_origins = await nameAnalysis.find_origins(req); - helper.log_to_file_queue(req.body.uuid, "[Done] Finding Origins") + else{ + await stringAnalysis.split_comma(req, all_words) } + if (req.body.option.includes("SplitWordsByUpperCase")) { helper.log_to_file_queue(req.body.uuid, "[Starting] Split by UpperCase") await stringAnalysis.split_upper_case(req, all_words) @@ -424,7 +466,7 @@ app.post("/analyze_string", async function(req, res, next) { if (user_info_normal.data.length > 0) { if (req.body.option.includes("ExtractMetadata")) { helper.log_to_file_queue(req.body.uuid, "[Starting] Network Graph") - graph = await visualize.visualize_force_graph(req.body.string, user_info_normal.data, "fast") + graph = await visualize.visualize_force_graph(req, user_info_normal.data, "fast") helper.log_to_file_queue(req.body.uuid, "[Done] Network Graph") } else { helper.log_to_file_queue(req.body.uuid, "[Warning] NetworkGraph needs ExtractMetadata") @@ -441,7 +483,8 @@ app.post("/analyze_string", async function(req, res, next) { helper.log_to_file_queue(req.body.uuid, "[Finished] Analyzing: " + req.body.string + " Task: " + req.body.uuid) - /*fs.writeFileSync('./test.json', JSON.stringify({ + + /*fs.writeFileSync('./test.json', JSON.stringify({ username: username, uuid: temp_uuid, info, diff --git a/modules/fast-scan.js b/modules/fast-scan.js index 96b5f4d..8b21552 100755 --- a/modules/fast-scan.js +++ b/modules/fast-scan.js @@ -9,7 +9,7 @@ var cheerio = require('cheerio'); var engine = require('./engine.js') async function find_username_normal(req) { - helper.log_to_file_queue(req.body.uuid, "[init] Selected websites: " + helper.websites_entries.filter((item) => item.selected == 'true').length) + helper.log_to_file_queue(req.body.uuid, "[init] Selected websites: " + helper.websites_entries.filter((item) => item.selected == 'true').length + " for username: " + req.body.string) const time = new Date(); var all_results = [] var [first_re_try, first_profiles] = await find_username_normal_wrapper(req, helper.websites_entries); @@ -40,6 +40,7 @@ async function get_failed(req, failed) { "method": "" }; temp_profile.method = "failed" + temp_profile.username = req.body.string temp_profile.link = site.url.replace("{username}", req.body.string) temp_result.push(temp_profile) }); @@ -151,6 +152,7 @@ async function find_username_site(uuid, username, options, site) { } } + temp_profile.username = username temp_profile.link = site.url.replace("{username}", username); temp_profile.type = site.type temp_profile.rank = site.global_rank diff --git a/modules/helper.js b/modules/helper.js index 37fcfa9..90b7583 100755 --- a/modules/helper.js +++ b/modules/helper.js @@ -26,6 +26,7 @@ var detection_level = { var profile_template = { "found": 0, + "username":"", "image": "", "link": "", "rate": "", diff --git a/modules/slow-scan.js b/modules/slow-scan.js index d9b8e50..dc3eed6 100755 --- a/modules/slow-scan.js +++ b/modules/slow-scan.js @@ -18,7 +18,7 @@ if (process.platform == 'win32') { } async function find_username_advanced(req) { - helper.log_to_file_queue(req.body.uuid, "[init] Selected websites: " + helper.websites_entries.filter((item) => item.selected == 'true').length) + helper.log_to_file_queue(req.body.uuid, "[init] Selected websites: " + helper.websites_entries.filter((item) => item.selected == 'true').length + " for username: " + req.body.string) const time = new Date(); const functions = []; var all_results = [] @@ -111,6 +111,7 @@ async function find_username_site(uuid, username, options, site) { helper.verbose && console.log(err); } + temp_profile.username = username temp_profile.text = sanitizeHtml(text_only); temp_profile.title = sanitizeHtml(title); temp_profile.language = language @@ -170,6 +171,7 @@ async function find_username_site(uuid, username, options, site) { temp_profile.type = site.type resolve(temp_profile); } else if (temp_profile.image != "") { + temp_profile.username = username temp_profile.text = "unavailable"; temp_profile.title = "unavailable"; temp_profile.language = "unavailable" diff --git a/modules/string-analysis.js b/modules/string-analysis.js index 3193d55..9ae395d 100755 --- a/modules/string-analysis.js +++ b/modules/string-analysis.js @@ -143,6 +143,16 @@ async function analyze_string(req, all_words) { helper.log_to_file_queue(req.body.uuid, "[Done] String analysis") } +async function split_comma(req, all_words) { + try { + req.body.string.split(",").forEach((item) => { + if (item.length > 1 && !all_words.unknown.includes(item) && !all_words.maybe.includes(item)) { + all_words.unknown.push(item.toLowerCase()); + } + }); + } catch (err) {} +} + async function split_upper_case(req, all_words) { try { req.body.string.match(/[A-Z][a-z]+/g).forEach((item) => { @@ -221,6 +231,7 @@ module.exports = { find_symbols, find_numbers, convert_numbers, + split_comma, split_upper_case, split_alphabet_case, most_common, diff --git a/modules/visualize.js b/modules/visualize.js index 59be678..08d5064 100755 --- a/modules/visualize.js +++ b/modules/visualize.js @@ -1,16 +1,34 @@ var ixora = require('ixora').QBIxora var helper = require('./helper.js'); -async function visualize_force_graph(username, detected, type) { +async function visualize_force_graph(req, detected, type) { try { var graph = new ixora('Social-Analyzer', false); temp_filtered = [] temp_filtered = detected.filter(item => item.status == "good") if (temp_filtered.length > 0) { - graph.add_node(username, search = username, _set = { - 'header': username - }) + if (req.body.group){ + + graph.add_node(req.body.string, search = req.body.string, _set = { + 'header': req.body.string + }) + + req.body.string.split(",").forEach( username => { + graph.add_node(username, search = username, _set = { + 'header': username + }) + + graph.add_edge(username, req.body.string, { + 'width': 1 + }) + }) + } + else{ + graph.add_node(req.body.string, search = req.body.string, _set = { + 'header': req.body.string + }) + } temp_filtered.forEach(site => { @@ -18,9 +36,10 @@ async function visualize_force_graph(username, detected, type) { 'header': site.link }) - graph.add_edge(username, site.link, { + graph.add_edge(site.username, site.link, { 'width': 1 }) + if ("metadata" in site) { if (site.metadata != "unavailable" && site.metadata.length > 0) { site.metadata.forEach(meta => { @@ -34,8 +53,8 @@ async function visualize_force_graph(username, detected, type) { temp_string = meta.property + " -> " + meta.content } - if (temp_string.length > 50) { - temp_string = temp_string.substring(0, 50).replace(/\r?\n|\r/g, "") + ".." + if (temp_string.length > 70) { + temp_string = temp_string.substring(0, 70).replace(/\r?\n|\r/g, "") + ".." } else { temp_string = temp_string.replace(/\r?\n|\r/g, "") } diff --git a/package-lock.json b/package-lock.json index a49d28f..fa5f18c 100755 --- a/package-lock.json +++ b/package-lock.json @@ -20,7 +20,7 @@ "generatorics": "1.1.0", "html-to-text": "6.0.0", "https-proxy-agent": "5.0.0", - "ixora": "1.0.6", + "ixora": "1.0.7", "langs": "2.0.0", "lodash": ">=4.17.21", "most-common-words-by-language": "2.17.8", @@ -1022,9 +1022,9 @@ "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" }, "node_modules/ixora": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/ixora/-/ixora-1.0.6.tgz", - "integrity": "sha512-+17pridJyYLWF/Bu6mYeqvILILAzByDTANswiEakcCuj8CQTHLLntoP1k9Lw3CSQ51YsMwbFBOJ4aFzagOFJfg==", + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/ixora/-/ixora-1.0.7.tgz", + "integrity": "sha512-3aUgabE+44Zta1HXd4feBwpkmTbAO1wJqRgLyzlsUMzhV1daIzkLoZHoVgXqGIqFPeZkjRpJQ43lZy0bB8DRwQ==", "dependencies": { "ejs": "3.1.6", "open": "8.2.1" @@ -2817,9 +2817,9 @@ "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" }, "ixora": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/ixora/-/ixora-1.0.6.tgz", - "integrity": "sha512-+17pridJyYLWF/Bu6mYeqvILILAzByDTANswiEakcCuj8CQTHLLntoP1k9Lw3CSQ51YsMwbFBOJ4aFzagOFJfg==", + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/ixora/-/ixora-1.0.7.tgz", + "integrity": "sha512-3aUgabE+44Zta1HXd4feBwpkmTbAO1wJqRgLyzlsUMzhV1daIzkLoZHoVgXqGIqFPeZkjRpJQ43lZy0bB8DRwQ==", "requires": { "ejs": "3.1.6", "open": "8.2.1" diff --git a/package.json b/package.json index 204d648..bcf14c8 100755 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ "generatorics": "1.1.0", "html-to-text": "6.0.0", "https-proxy-agent": "5.0.0", - "ixora": "1.0.6", + "ixora": "1.0.7", "langs": "2.0.0", "lodash": ">=4.17.21", "most-common-words-by-language": "2.17.8", diff --git a/public/app.html b/public/app.html index 9c8f2a7..ba47427 100755 --- a/public/app.html +++ b/public/app.html @@ -643,7 +643,7 @@ text-align: right; } - #div-right-icon { + .div-right-icon { float: right; font-size: 16px; } @@ -755,7 +755,7 @@
Enter Profile Name
-
+
@@ -775,7 +775,7 @@
String LookUps
-
+
@@ -786,7 +786,7 @@
Most common words by languages
-
+
@@ -794,7 +794,7 @@
Extracted names with orginis
-
+
@@ -802,7 +802,7 @@
Extracted words information
-
+
@@ -810,7 +810,7 @@
Custom Search
-
+
@@ -818,14 +818,14 @@
Possible words for combinations
-
+
Save Report
-
+
@@ -838,35 +838,35 @@
Profiles
-
+
Detected Profiles (Advanced)
-
+
Detected Profiles (Normal)
-
+
All Profiles (Normal)
-
+
Graph
-
+
@@ -874,7 +874,7 @@
Stats
-
+
@@ -882,14 +882,14 @@
Logs
-
+
Save to file as
-
+
@@ -1280,6 +1280,7 @@ url: "/analyze_string", data: { 'string': $('#string-to-analyze').val(), + 'group':false, 'option': list_of_options.join(","), 'uuid': random_string }, @@ -1333,7 +1334,7 @@ temp_image = '' } var temp_extract = add_extract(site); - temp_tr += '