fix wording and update web based script

This commit is contained in:
Jieyab89
2026-08-08 07:34:57 +07:00
parent 2f5b82778b
commit 44d26c9c47
4 changed files with 209 additions and 26 deletions
+7 -10
View File
@@ -8,8 +8,8 @@ import time
urls = {
"readme": "https://raw.githubusercontent.com/Jieyab89/OSINT-Cheat-sheet/refs/heads/main/README.md",
"wiki": "https://github.com/Jieyab89/OSINT-Cheat-sheet/wiki",
"articles": "https://raw.githubusercontent.com/Jieyab89/OSINT-Cheat-sheet/main/awesome-article.md"
#"scripts": "https://github.com/Jieyab89/OSINT-Cheat-sheet/blob/main/Script/README.md" soon
"articles": "https://raw.githubusercontent.com/Jieyab89/OSINT-Cheat-sheet/main/awesome-article.md",
"scripts": "https://raw.githubusercontent.com/Jieyab89/OSINT-Cheat-sheet/main/Script/README.md"
}
headers = {
@@ -37,11 +37,10 @@ current_category = None
def parse_markdown_md(url):
time.sleep(6)
response = requests.get(url, headers=headers, timeout=30)
# response = requests.get(url)
response = requests.get(url, headers=headers, timeout=30)
time.sleep(6)
response = requests.get(url, headers=headers, timeout=30)
if response.status_code != 200:
print(f"[-] Failed to fetch: {url}")
return {}
@@ -66,12 +65,8 @@ def parse_markdown_md(url):
return parsed
def parse_github_wiki(url):
wiki_items = []
time.sleep(6)
response = requests.get(url, headers=headers, timeout=30)
wiki_items = []
# response = requests.get(url)
time.sleep(6)
response = requests.get(url, headers=headers, timeout=30)
@@ -79,6 +74,7 @@ def parse_github_wiki(url):
print(f"[-] Failed to fetch: {url}")
return {}
wiki_items = []
soup = BeautifulSoup(response.text, "html.parser")
for link in soup.select(".wiki-pages-box a"):
title = link.text.strip()
@@ -91,6 +87,7 @@ def parse_github_wiki(url):
new_categories.update(parse_markdown_md(urls["readme"]))
new_categories.update(parse_markdown_md(urls["articles"]))
new_categories.update(parse_markdown_md(urls["scripts"]))
new_categories.update(parse_github_wiki(urls["wiki"]))
for category, new_data in new_categories.items():
@@ -105,4 +102,4 @@ for category, new_data in new_categories.items():
with open(json_file, "w", encoding="utf-8") as f:
json.dump(list(existing_categories.values()), f, indent=4, ensure_ascii=False)
print(f"[+] Data updated: {json_file}")
print(f"[+] Data updated: {json_file}")