From 44d26c9c475ff52fb1d3f96740d45b403c74be17 Mon Sep 17 00:00:00 2001 From: Jieyab89 Date: Sat, 8 Aug 2026 07:34:57 +0700 Subject: [PATCH] fix wording and update web based script --- README.md | 2 +- Script/SOCMINT-Twitter/Readme.md | 30 ++--- Web-Based/gen-node/node-generate.py | 17 ++- Web-Based/osint_data.json | 186 ++++++++++++++++++++++++++++ 4 files changed, 209 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 8909f9e..f51b961 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,7 @@ You can build it with VM or Live USB make sure you have sandbox machine # OSINT Academy -- [Jieyab Gitbook OSIN and other bracnh Intel](https://github.com/Jieyab89/OSINT-Cheat-sheet/wiki/1-Notes-Jieyab-Wiki-and-Gitbook) +- [Jieyab Gitbook OSINT and other bracnh Intel](https://github.com/Jieyab89/OSINT-Cheat-sheet/wiki/1-Notes-Jieyab-Wiki-and-Gitbook) - [Maltego academy](https://academy.maltego.com/) - [Darkweb academy](https://www.darkwebacademy.com/labs/) diff --git a/Script/SOCMINT-Twitter/Readme.md b/Script/SOCMINT-Twitter/Readme.md index 2bd1b90..15a8ce5 100644 --- a/Script/SOCMINT-Twitter/Readme.md +++ b/Script/SOCMINT-Twitter/Readme.md @@ -2,6 +2,21 @@ Image +# Update Note + +1. Update infinity scroll and load new data for each search also in graph +2. Update data corelation +3. Fix business logic flow +4. Monitoring (Soon) +5. MCP (Soon) +6. Add more parameter for enrichment +7. Add no rate limit (throttle) +8. Add Google CSE data source +9. Expand data user profile post, follower and following, reply post, retweet post in graph — +10. Add sentiment analysis for clustering data, pro, neutral, con based on archive data and dump data +11. Add more data source and other parameter (soon) still research +12. Add more detail data source for the context + ## Features - **Multi-source search** — one query fans out to Cookie, Xquik API, @@ -153,21 +168,6 @@ banner reflects it. Neither backend is a ground-truth classifier — short text, sarcasm, and irony degrade accuracy either way. Treat results as a starting point for investigation, not a verdict. -## Update Note - -1. Update infinity scroll and load new data for each search also in graph -2. Update data corelation -3. Fix business logic flow -4. Monitoring (Soon) -5. MCP (Soon) -6. Add more parameter for enrichment -7. Add no rate limit (throttle) -8. Add Google CSE data source -9. Expand data user profile post, follower and following, reply post, retweet post in graph — -10. Add sentiment analysis for clustering data, pro, neutral, con based on archive data and dump data -11. Add more data source and other parameter (soon) still research -12. Add more detail data source for the context - ## Settup ```bash diff --git a/Web-Based/gen-node/node-generate.py b/Web-Based/gen-node/node-generate.py index e86f9a5..96e8dfd 100644 --- a/Web-Based/gen-node/node-generate.py +++ b/Web-Based/gen-node/node-generate.py @@ -8,8 +8,8 @@ import time urls = { "readme": "https://raw.githubusercontent.com/Jieyab89/OSINT-Cheat-sheet/refs/heads/main/README.md", "wiki": "https://github.com/Jieyab89/OSINT-Cheat-sheet/wiki", - "articles": "https://raw.githubusercontent.com/Jieyab89/OSINT-Cheat-sheet/main/awesome-article.md" - #"scripts": "https://github.com/Jieyab89/OSINT-Cheat-sheet/blob/main/Script/README.md" soon + "articles": "https://raw.githubusercontent.com/Jieyab89/OSINT-Cheat-sheet/main/awesome-article.md", + "scripts": "https://raw.githubusercontent.com/Jieyab89/OSINT-Cheat-sheet/main/Script/README.md" } headers = { @@ -37,11 +37,10 @@ current_category = None def parse_markdown_md(url): time.sleep(6) - response = requests.get(url, headers=headers, timeout=30) - # response = requests.get(url) + response = requests.get(url, headers=headers, timeout=30) time.sleep(6) response = requests.get(url, headers=headers, timeout=30) - + if response.status_code != 200: print(f"[-] Failed to fetch: {url}") return {} @@ -66,12 +65,8 @@ def parse_markdown_md(url): return parsed def parse_github_wiki(url): - wiki_items = [] - time.sleep(6) response = requests.get(url, headers=headers, timeout=30) - wiki_items = [] - # response = requests.get(url) time.sleep(6) response = requests.get(url, headers=headers, timeout=30) @@ -79,6 +74,7 @@ def parse_github_wiki(url): print(f"[-] Failed to fetch: {url}") return {} + wiki_items = [] soup = BeautifulSoup(response.text, "html.parser") for link in soup.select(".wiki-pages-box a"): title = link.text.strip() @@ -91,6 +87,7 @@ def parse_github_wiki(url): new_categories.update(parse_markdown_md(urls["readme"])) new_categories.update(parse_markdown_md(urls["articles"])) +new_categories.update(parse_markdown_md(urls["scripts"])) new_categories.update(parse_github_wiki(urls["wiki"])) for category, new_data in new_categories.items(): @@ -105,4 +102,4 @@ for category, new_data in new_categories.items(): with open(json_file, "w", encoding="utf-8") as f: json.dump(list(existing_categories.values()), f, indent=4, ensure_ascii=False) -print(f"[+] Data updated: {json_file}") +print(f"[+] Data updated: {json_file}") \ No newline at end of file diff --git a/Web-Based/osint_data.json b/Web-Based/osint_data.json index 31ffb6c..1283f8e 100644 --- a/Web-Based/osint_data.json +++ b/Web-Based/osint_data.json @@ -63,6 +63,10 @@ { "name": "Darkweb academy", "url": "https://www.darkwebacademy.com/labs/" + }, + { + "name": "Jieyab Gitbook OSINT and other bracnh Intel", + "url": "https://github.com/Jieyab89/OSINT-Cheat-sheet/wiki/1-Notes-Jieyab-Wiki-and-Gitbook" } ] }, @@ -88,6 +92,10 @@ { "name": "Exif python script remover", "url": "https://github.com/davidbombal/red-python-scripts/blob/main/remove_exif.py" + }, + { + "name": "Metadata Remover", + "url": "https://metadataremover.ai/metadata-viewer" } ] }, @@ -1674,6 +1682,10 @@ { "name": "DomScan", "url": "https://domscan.net/tools/security" + }, + { + "name": "CC.LA", + "url": "https://cc.la" } ] }, @@ -6988,6 +7000,14 @@ { "name": "PeekVault", "url": "https://peekvault.com/" + }, + { + "name": "Jieyab ft Xquik Twitter Scrappe and MCP and monitoring", + "url": "https://github.com/Jieyab89/OSINT-Cheat-sheet/tree/main/Script/SOCMINT-Twitter" + }, + { + "name": "trends24 whats trend in X arround world", + "url": "https://trends24.in/indonesia/" } ] }, @@ -22218,6 +22238,172 @@ { "name": "2 Notes Jieyab Wiki and Gitbook", "url": "https://github.com/Jieyab89/OSINT-Cheat-sheet/wiki/2-Notes-Jieyab-Wiki-and-Gitbook" + }, + { + "name": "Update", + "url": "https://github.com/Jieyab89/OSINT-Cheat-sheet/wiki#update" + }, + { + "name": "1 Notes Jieyab Wiki and Gitbook", + "url": "https://github.com/Jieyab89/OSINT-Cheat-sheet/wiki/1-Notes-Jieyab-Wiki-and-Gitbook" + } + ] + }, + { + "category": "SIGINT", + "items": [ + { + "name": "osmocom", + "url": "https://osmocom.org/projects/baseband/wiki/Hardware" + }, + { + "name": "Motorola", + "url": "https://osmocom.org/projects/baseband/wiki/MobistelEL570" + }, + { + "name": "Hackrf", + "url": "https://greatscottgadgets.com/hackrf/" + }, + { + "name": "bladerf", + "url": "https://www.nuand.com/bladerf-1/" + }, + { + "name": "rtl sdr", + "url": "https://www.rtl-sdr.com/" + }, + { + "name": "Tenda", + "url": "https://www.tendacn.com/material/show/2002565" + }, + { + "name": "Wifi Pineaple HAK5", + "url": "https://shop.hak5.org/products/wifi-pineapple" + }, + { + "name": "tp link", + "url": "https://www.tp-link.com/id/" + }, + { + "name": "Walkie-talkie", + "url": "https://en.wikipedia.org/wiki/Walkie-talkie" + }, + { + "name": "flipper", + "url": "https://flipper.net/" + }, + { + "name": "raspberrypi", + "url": "https://www.raspberrypi.com/" + }, + { + "name": "wireshark", + "url": "https://www.wireshark.org/" + }, + { + "name": "kismetwireless", + "url": "https://www.kismetwireless.net/" + }, + { + "name": "aircrack", + "url": "https://www.aircrack-ng.org/" + }, + { + "name": "nmap", + "url": "https://nmap.org/" + }, + { + "name": "netmonster", + "url": "https://play.google.com/store/apps/details?id=cz.mroczis.netmonster" + }, + { + "name": "wigle", + "url": "https://wigle.net/" + }, + { + "name": "angryip", + "url": "https://angryip.org/" + }, + { + "name": "advanced-ip-scanner", + "url": "https://www.advanced-ip-scanner.com/id/" + }, + { + "name": "pysim", + "url": "https://github.com/osmocom/pysim" + }, + { + "name": "osmocom", + "url": "https://osmocom.org/projects/baseband" + }, + { + "name": "G Maps APIs", + "url": "https://mapsplatform.google.com/lp/maps-apis/" + }, + { + "name": "leafletjs", + "url": "https://leafletjs.com/" + }, + { + "name": "arcgis", + "url": "https://www.arcgis.com/apps/mapviewer/index.html" + }, + { + "name": "d3js", + "url": "https://d3js.org/" + } + ] + }, + { + "category": "Query & Script List", + "items": [] + }, + { + "category": "Report", + "items": [] + }, + { + "category": "Jieyab89 Scripts", + "items": [ + { + "name": "CNN-News-Scrapping-and-Archive", + "url": "https://github.com/Jieyab89/OSINT-Cheat-sheet/tree/main/Script/CNN-News-Scrapping-and-Archive" + }, + { + "name": "Darkweb-Scrapping", + "url": "https://github.com/Jieyab89/OSINT-Cheat-sheet/tree/main/Script/Darkweb-Scrapping" + }, + { + "name": "Dorking", + "url": "https://github.com/Jieyab89/OSINT-Cheat-sheet/tree/main/Script/Dorking" + }, + { + "name": "Github-Search", + "url": "https://github.com/Jieyab89/OSINT-Cheat-sheet/tree/main/Script/Github-Search" + }, + { + "name": "Linkedin-Script", + "url": "https://github.com/Jieyab89/OSINT-Cheat-sheet/tree/main/Script/Linkedin-Script" + }, + { + "name": "OSM-QUERY-LIST", + "url": "https://github.com/Jieyab89/OSINT-Cheat-sheet/tree/main/Script/OSM-QUERY-LIST" + }, + { + "name": "Red-Team", + "url": "https://github.com/Jieyab89/OSINT-Cheat-sheet/tree/main/Script/Red-Team" + }, + { + "name": "SENTINEL-SAT-SCRIPT-LIST", + "url": "https://github.com/Jieyab89/OSINT-Cheat-sheet/tree/main/Script/SENTINEL-SAT-SCRIPT-LIST" + }, + { + "name": "SOCMINT-Twitter", + "url": "https://github.com/Jieyab89/OSINT-Cheat-sheet/tree/main/Script/SOCMINT-Twitter" + }, + { + "name": "Web-TLD-Enumerate-NS-Check", + "url": "https://github.com/Jieyab89/OSINT-Cheat-sheet/tree/main/Script/Web-TLD-Enumerate-NS-Check" } ] }