WEBDIENSTE/osmedeus
1
0
Fork 0
mirror of https://github.com/j3ssie/osmedeus.git synced 2026-09-13 05:07:45 +02:00
Code Issues Packages Projects Releases Wiki Activity

Fix and update some bugs

Browse Source 
Signed-off-by: j3ssie <jeromej3m@gmail.com>
This commit is contained in:
j3ssie
2019-12-01 14:09:16 +07:00
parent 41350b581f
commit a438649ece
15 changed files with 226 additions and 184 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Dockerfile
+18 -35
View File
@@ -1,38 +1,21 @@
FROM debian:latest
RUN apt-get update && \
apt-get -qq install \
npm \
locales \
git \
sudo \
wget \
python3-pip \
python-pip \
curl \
libcurl4-openssl-dev \
bsdmainutils \
xsltproc \
build-essential
# Set the locale
RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \
dpkg-reconfigure --frontend=noninteractive locales && \
update-locale LANG=en_US.UTF-8
ENV LANG en_US.UTF-8
RUN cp -av /usr/bin/pip2 /usr/bin/pip2.7 && \
pip install setuptools && \
pip3 install setuptools && \
pip install wheel && \
pip3 install wheel
COPY . /home/Osmedeus
FROM debian:buster-20191014-slim
ARG OSMEDEUS_VERSION=v.2.1
RUN sed -i 's/main/main contrib non-free/' /etc/apt/sources.list
WORKDIR /home/Osmedeus
RUN ./install.sh && \
ENV LANG="en_US.UTF-8" \
LANGUAGE="en_US:en" \
LC_ALL="en_US.UTF-8"
RUN apt-get update && \
apt-get -yu dist-upgrade && \
apt-get -qq install npm && \
apt-get -qq install locales git sudo wget python3-pip python-pip curl libcurl4-openssl-dev bsdmainutils xsltproc && \
sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \
locale-gen && \
cp -av /usr/bin/pip2 /usr/bin/pip2.7 && \
git clone --depth 1 https://github.com/j3ssie/Osmedeus -b $OSMEDEUS_VERSION . && \
./install.sh && \
go get -u github.com/tomnomnom/unfurl && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
ENTRYPOINT ["python3", "server/manage.py", "runserver", "0.0.0.0:8000"]
EXPOSE 8000
CMD ["./osmedeus.py"]
README.md
+14 -11
View File
@@ -25,6 +25,10 @@ cd Osmedeus
This install only focus on Kali linux, check more install on [Usage page](https://j3ssie.github.io/Osmedeus/installation/)
## Using Docker
Check out [docker-osmedeus](https://github.com/mablanco/docker-osmedeus) by [mabnavarrete](https://twitter.com/mabnavarrete)
## How to use
If you have no idea what are you doing just type the command below or check out the [Advanced Usage](https://j3ssie.github.io/Osmedeus/advanced/)
@@ -64,10 +68,6 @@ If you love my tool and would like to support my work. [Become a backer](https:/
## Demo
[![asciicast](https://asciinema.org/a/266599.svg)](https://asciinema.org/a/266599)
![Reports](imgs/osmedeus-report.png)
![Slack Noti](imgs/slack_noti.png)
![Web UI 1](imgs/ui-1.png)
![Web UI 3](imgs/ui-3.png)
@@ -89,8 +89,6 @@ If you love my tool and would like to support my work. [Become a backer](https:/
./osmedeus.py -m portscan -i "1.2.3.4/24"
./osmedeus.py -m "portscan,vulnscan" -i "1.2.3.4/24" -w result_folder
./osmedeus.py -m "git" -i 'repo:https://github.com/foo/bar'
./osmedeus.py -m "git" -i 'user:sample'
# direct list mode examples
./osmedeus.py -m portscan -I list_of_targets.txt
@@ -98,14 +96,19 @@ If you love my tool and would like to support my work. [Become a backer](https:/
./osmedeus.py -m screen -I list_of_targets.txt -w result_folder
# report mode
./osemdeus.py -t example.com --report list
./osemdeus.py -t example.com --report export
./osemdeus.py -t example.com --report sum
./osemdeus.py -t example.com --report short
./osemdeus.py -t example.com --report full
./osmedeus.py -t example.com --report list
./osmedeus.py -t example.com --report export
./osmedeus.py -t example.com --report sum
./osmedeus.py -t example.com --report short
./osmedeus.py -t example.com --report full
```
## What to do next after do recon?
Check out my scanner [Jaeles](https://github.com/jaeles-project/jaeles).
## Disclaimer
Most of this tool done by the authors of the tool that list in [CREDITS.md](https://github.com/j3ssie/Osmedeus/blob/master/CREDITS.md).
imgs/osmedeus-architecture.png
BIN
View File
Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 716 KiB

After

Width:  |  Height:  |  Size: 723 KiB

install.sh
+13 -10
View File
@@ -23,12 +23,13 @@ install_banner "git, nmap, masscan, chromium, npm, golang"
[ -x "$(command -v go)" ] || sudo $PACKGE_MANAGER install golang -y 2>/dev/null
[ -x "$(command -v make)" ] || sudo $PACKGE_MANAGER install build-essential -y 2>/dev/null
[ -x "$(command -v csvlook)" ] || sudo $PACKGE_MANAGER install csvkit -y 2>/dev/null
[ -x "$(command -v ripgrep)" ] || sudo $PACKGE_MANAGER install ripgrep -y 2>/dev/null
[ -x "$(command -v rg)" ] || sudo $PACKGE_MANAGER install ripgrep -y 2>/dev/null
[ -x "$(command -v unzip)" ] || sudo $PACKGE_MANAGER install unzip -y 2>/dev/null
[ -x "$(command -v chromium-browser)" ] || sudo $PACKGE_MANAGER install chromium-browser -y 2>/dev/null
[ -x "$(command -v xsltproc)" ] || sudo $PACKGE_MANAGER install xsltproc -y 2>/dev/null
[ -x "$(command -v make)" ] || sudo $PACKGE_MANAGER install build-essential -y 2>/dev/null
[ -x "$(command -v pip)" ] || sudo $PACKGE_MANAGER install python-pip -y 2>/dev/null
[ -x "$(command -v pip3)" ] || sudo $PACKGE_MANAGER install python3-pip -y 2>/dev/null
[ -x "$(command -v xsltproc)" ] || sudo $PACKGE_MANAGER install xsltproc -y 2>/dev/null
pip install setuptools 2>/dev/null
pip3 install setuptools 2>/dev/null
@@ -128,6 +129,7 @@ python3 server/manage.py migrate
python3 server/manage.py makemigrations api
python3 server/manage.py migrate api
python3 scripts/init.py
python3 scripts/reload.py
### adding gopath if GOPATH not in default shellrc
if ! grep -Fxq "GOPATH" "$DEFAULT_SHELL"; then
@@ -148,8 +150,6 @@ GO_BIN=$(which go)
##
# Install go stuff
##
install_banner "amass"
$GO_BIN get -u github.com/OWASP/Amass/...
install_banner "subfinder"
$GO_BIN get -u github.com/subfinder/subfinder
install_banner "gobuster"
@@ -188,9 +188,15 @@ install_banner "ffuf"
$GO_BIN get -u github.com/ffuf/ffuf
install_banner "rgf"
$GO_BIN get -u github.com/j3ssie/rgf
install_banner "jaeles"
$GO_BIN get -u github.com/jaeles-project/jaeles
install_banner "go cli-utils"
$GO_BIN get -u github.com/j3ssie/go-auxs/getIP
$GO_BIN get -u github.com/j3ssie/go-auxs/just-resolved
$GO_BIN get -u github.com/j3ssie/go-auxs/cinfo
install_banner "amass"
export GO111MODULE=on
go get -u github.com/OWASP/Amass/v3/...
cp $GO_DIR/* "$PLUGINS_PATH/go/" 2>/dev/null
# install_banner "observatory"
@@ -221,7 +227,7 @@ chmod +x $PLUGINS_PATH/findomain
# Install python stuff
##
install_banner "truffleHog, wfuzz"
install_banner "truffleHog"
pip install truffleHog
cd $PLUGINS_PATH
@@ -238,12 +244,9 @@ install_banner "Metabigor"
git clone https://github.com/j3ssie/Metabigor 2>/dev/null
pip3 install -r Metabigor/requirements.txt
install_banner "bass"
git clone https://github.com/Abss0x7tbh/bass 2>/dev/null
pip3 install -r bass/requirements.txt
install_banner "dirsearch"
git clone https://github.com/maurosoria/dirsearch 2>/dev/null
install_banner "Arjun"
git clone https://github.com/s0md3v/Arjun 2>/dev/null
lib/alias/dirscan
+2 -2
View File
@@ -38,10 +38,10 @@ def routine(args):
# init the summary
if not utils.not_empty_file(f'{final_summary}'):
head = "input,position,status_code,content_length,content_words\n"
head = "FUZZ,url,redirectlocation,position,status_code,content_length,content_words,content_lines\n"
utils.just_write(final_summary, head)
sep = f"\n{target},,,,\n"
sep = f"{target},,,,\n"
utils.just_append(final_summary, sep + "\n".join(data))
lib/alias/format_input
+18 -1
View File
@@ -3,6 +3,7 @@ import os
import sys
import time
import argparse
import re
import urllib.parse
sys.path.append(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
@@ -10,6 +11,16 @@ from core import utils
# portscan -i /tmp/target.txt -o '$WORKSPACE/portscan/$OUTPUT' -s '$WORKSPACE/portscan/summary.txt' -p '$PLUGINS_PATH'
def grep_the_IP(data):
cidr_regex = "((\d){1,3}\.){3}(\d){1,3}(\/(\d){1,3})?"
ips = []
p = re.compile(cidr_regex)
for m in p.finditer(data):
ips.append(m.group())
if ips:
return ips[0]
return ips
def routine(args):
target = args.input
output = args.output
@@ -23,7 +34,13 @@ def routine(args):
f'cat {target} | unfurl -u format https://%d:%P >> {output}-port.txt')
os.system(f'cat {target} | unfurl -u format %d >> {output}-domains.txt')
os.system(f'cat {target} | unfurl -u format %r >> {output}-root.txt')
time.sleep(0.3)
ips = []
data = utils.just_read(target, get_list=True)
for line in data:
ips.append(grep_the_IP(line))
utils.just_write(f"{output}-range.txt", "\n".join(ips))
time.sleep(0.2)
parser = argparse.ArgumentParser(description="VulnScan alias")
lib/alias/paramfinder
Executable
+45
View File
@@ -0,0 +1,45 @@
#!/usr/bin/env python3
import os
import sys
import time
import argparse
import urllib.parse
sys.path.append(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
from core import utils
# $ALIAS_PATH/paramfinding -i '[[0]]' -o '$WORKSPACE/params/raw' -s '$WORKSPACE/params/summary-$OUTPUT.txt' -p '$PLUGINS_PATH'
def routine(args):
target = args.input
output = args.output
strip_target = utils.strip_slash(target)
plugin = args.plugin
summary = args.summary
cmd = f'python3 {plugin}/Arjun/arjun.py -t 40 -f {plugin}/Arjun/db/params.txt --get -u {target} > {output}/{strip_target}.txt'
utils.print_info(f"Execute: {cmd}")
os.system(cmd)
contents = utils.just_read(f'{output}/{strip_target}.txt')
if 'parameter found:' in contents or 'potential parameter:' in contents:
params = []
for line in contents.split("\n"):
if 'Valid parameter found: ' in line:
params.append(line.split(': ')[1])
if 'potential parameter: ' in line:
params.append(line.split(': ')[1])
data = f"{target} | params: " + ", ".join(list(set(params))) + "\n"
# append to summary
utils.just_append(summary, data)
time.sleep(0.3)
parser = argparse.ArgumentParser(description="LinkFinder alias")
parser.add_argument('-i', '--input', action='store', dest='input', help='input')
parser.add_argument('-o', '--output', action='store', dest='output', help='output')
parser.add_argument('-p', '--plugin', action='store', dest='plugin', help='plugin')
parser.add_argument('-s', '--summary', action='store',
dest='summary', help='summary')
args = parser.parse_args()
routine(args)
lib/client/helpers.py
+10 -10
View File
@@ -88,8 +88,8 @@ python3 osmedeus.py -w sample2 -m dirb -I /tmp/list_of_hosts.txt
-s, --slow "subdomain"
Only running slow mode in subdomain module
--noslack Disable slack notification
--nomonitor Disable monitor new assets
--slack Enable slack notification
--monitor Enable monitor new assets
--debug Just for debug purpose
'''.format(G, GR, B))
sys.exit(0)
@@ -118,15 +118,15 @@ gitscan, cors, ipspace
{1}[{0}Report Usage{1}]{1}
===================
./osemdeus.py --report <mode> -t <workspace> [-m <module>]
./osmedeus.py --report <mode> -t <workspace> [-m <module>]
{1}[{0}Example Commands{1}]{1}
===================
./osemdeus.py -t example.com --report list
./osemdeus.py -t example.com --report sum
./osemdeus.py -t example.com --report path
./osemdeus.py -t example.com --report short
./osemdeus.py -t example.com -m subdomain --report short
./osemdeus.py -t example.com -m subdomain, portscan --report short
./osemdeus.py -t example.com -m subdomain, portscan --report full
./osmedeus.py -t example.com --report list
./osmedeus.py -t example.com --report sum
./osmedeus.py -t example.com --report path
./osmedeus.py -t example.com --report short
./osmedeus.py -t example.com -m subdomain --report short
./osmedeus.py -t example.com -m subdomain, portscan --report short
./osmedeus.py -t example.com -m subdomain, portscan --report full
'''.format(G, GR, B))
lib/core/utils.py
+1 -1
View File
@@ -126,7 +126,7 @@ def print_elapsed(options):
def print_debug(text, options=None):
if not options:
return
if options.get('DEBUG'):
if options.get('DEBUG', False) or options.get('debug', False):
print(G + "#" * 20 + GR)
print(text)
print("#" * 20)
lib/mode/direct.py
-4
View File
@@ -32,12 +32,8 @@ def handle(options):
if utils.loop_grep(modules, 'sub'):
subdomain.SubdomainScanning(options)
# vhosts.VhostScan(options)
# permutation.PermutationScan(options)
probing.Probing(options)
fingerprint.Fingerprint(options)
# stoscan.StoScan(options)
# screenshot.ScreenShot(options)
if utils.loop_grep(modules, 'screen'):
screenshot.ScreenShot(options)
lib/mode/general.py
-4
View File
@@ -4,8 +4,6 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
from lib.core import utils
from modules import subdomain
from modules import vhosts
from modules import permutation
from modules import probing
from modules import formatting
from modules import fingerprint
@@ -33,8 +31,6 @@ def handle(options):
def single_handle(options):
subdomain.SubdomainScanning(options)
# vhosts.VhostScan(options)
# permutation.PermutationScan(options)
probing.Probing(options)
formatting.Formatting(options)
corscan.CORScan(options)
lib/mode/routine.py
+2 -1
View File
@@ -18,7 +18,8 @@ def routine_handle(options):
utils.print_target(options.get('TARGET'))
# move last result to monitor path and add options to compare
if options.get('MONITOR'):
# only do monitor on general mode
if options.get('MONITOR') and options['MODE'] == "general":
options['COMPARE_PATH'] = backup.init_backup(options)
utils.print_good("Created compare workspaces: {0}".format(
options['COMPARE_PATH']))
lib/workflow/direct.py
+31 -8
View File
@@ -5,7 +5,6 @@ cpu_cores = multiprocessing.cpu_count()
threads = str(cpu_cores * 2)
class SubdomainScanning:
reports = [
{
@@ -19,7 +18,7 @@ class SubdomainScanning:
'general': [
{
"banner": "Amass",
"cmd": "$GO_PATH/amass enum -timeout 5 -active -max-dns-queries 10000 -include-unresolvable -active -dir $WORKSPACE/subdomain/amass-$OUTPUT -d $TARGET -o $WORKSPACE/subdomain/$OUTPUT-amass.txt",
"cmd": "$GO_PATH/amass enum -timeout 10 -active -max-dns-queries 10000 -include-unresolvable -active -dir $WORKSPACE/subdomain/amass-$OUTPUT -d $TARGET -o $WORKSPACE/subdomain/$OUTPUT-amass.txt",
"output_path": "$WORKSPACE/subdomain/$OUTPUT-amass.txt",
"std_path": "$WORKSPACE/subdomain/std-$TARGET-amass.std"
},
@@ -120,6 +119,11 @@ class Probing:
"type": "bash",
"note": "final, slack, diff",
},
{
"path": "$WORKSPACE/probing/raw-allmassdns.txt",
"type": "bash",
"note": "final",
},
{
"path": "$WORKSPACE/probing/resolved-$OUTPUT.txt",
"type": "bash",
@@ -233,7 +237,7 @@ class Fingerprint:
{
"requirement": "$WORKSPACE/fingerprint/responses/index",
"banner": "rgf extract all",
"cmd": "$GO_PATH/rgf -d $WORKSPACE/fingerprint/responses/ | tee $WORKSPACE/fingerprint/rgf-all-$OUTPUT.txt",
"cmd": "$GO_PATH/rgf -dir $WORKSPACE/fingerprint/responses/ | tee $WORKSPACE/fingerprint/rgf-all-$OUTPUT.txt",
"output_path": "$WORKSPACE/fingerprint/rgf-all-$OUTPUT.txt",
"std_path": "",
},
@@ -421,15 +425,20 @@ class PortScan:
"type": "html",
"note": "final",
},
{
"path": "$WORKSPACE/portscan/screenshot/$OUTPUT-raw-gowitness.html",
"type": "html",
"note": "",
},
]
logs = []
commands = {
'general': [
{
"requirement": "$WORKSPACE/probing/ip-$OUTPUT.txt",
# "requirement": "$WORKSPACE/formatted/ip-$OUTPUT.txt",
"banner": "Masscan 65535 ports",
"cmd": "$ALIAS_PATH/portscan -i $WORKSPACE/probing/ip-$OUTPUT.txt -o '$WORKSPACE/portscan/$OUTPUT' -s '$WORKSPACE/portscan/summary.txt' -p '$PLUGINS_PATH'",
"output_path": "$WORKSPACE/portscan/$OUTPUT.csv",
"cmd": "$ALIAS_PATH/portscan -i $TARGET -o '$WORKSPACE/portscan/$OUTPUT' -s '$WORKSPACE/portscan/summary.txt' -p '$PLUGINS_PATH'",
"output_path": "$WORKSPACE/portscan/$OUTPUT.xml",
"std_path": "",
"waiting": "first",
},
@@ -444,18 +453,32 @@ class PortScan:
},
{
"requirement": "$WORKSPACE/portscan/$OUTPUT.csv",
"banner": "CSV beautify",
"banner": "Httprode new port",
"cmd": '''cat $WORKSPACE/portscan/$OUTPUT.csv | awk -F',' '{print $1":"$4}' | httprobe -c 30 | tee $WORKSPACE/portscan/http-$OUTPUT.txt''',
"output_path": "$WORKSPACE/portscan/http-$OUTPUT.txt",
"std_path": "",
},
{
"requirement": "$WORKSPACE/portscan/http-$OUTPUT.txt",
"banner": "aquatone",
"cmd": f"cat $WORKSPACE/portscan/http-$OUTPUT.txt | $GO_PATH/aquatone -screenshot-timeout 50000 -threads {threads} -out $WORKSPACE/portscan/$OUTPUT-aquatone",
"output_path": "$WORKSPACE/portscan/$OUTPUT-aquatone/aquatone_report.html",
"std_path": "$WORKSPACE/portscan/std-$OUTPUT-aquatone.std",
"waiting": "last",
},
# {
# "requirement": "$WORKSPACE/portscan/$OUTPUT.csv",
# "banner": "Screenshot on ports found",
# "cmd": "$GO_PATH/gowitness file -s $WORKSPACE/portscan/scheme-$OUTPUT.txt -t 30 --log-level fatal --destination $WORKSPACE/portscan/screenshot/raw-gowitness/ --db $WORKSPACE/portscan/screenshot/gowitness.db",
# "output_path": "$WORKSPACE/portscan/screenshot/gowitness.db",
# "std_path": "",
# "post_run": "clean_gowitness",
# "pre_run": "get_scheme",
# "cleaned_output": "$WORKSPACE/portscan/screenshot-$OUTPUT.html",
# "waiting": "last",
# },
],
}
@@ -525,7 +548,7 @@ class IPSpace:
'general': [
{
"banner": "Metabigor IP Lookup",
"cmd": "$PLUGINS_PATH/Metabigor/metabigor.py -m ip -t $TARGET -o $WORKSPACE/ipspace/range-$OUTPUT.txt",
"cmd": "PLUGINS_PATH/Metabigor/metabigor.py -m ip -t $TARGET -o $WORKSPACE/ipspace/range-$OUTPUT.txt",
"output_path": "$WORKSPACE/ipspace/range-$OUTPUT.txt",
"std_path": "",
"post_run": "get_amass",
lib/workflow/direct_list.py
+35 -17
View File
@@ -11,7 +11,7 @@ class Formatting:
"type": "bash"
},
{
"path": "$WORKSPACE/formatted/ip-$OUTPUT.txt",
"path": "$WORKSPACE/formatted/$OUTPUT-range.txt",
"type": "bash"
}
]
@@ -29,8 +29,8 @@ class Formatting:
{
"banner": "Resolve IP",
"requirement": "$WORKSPACE/formatted/$OUTPUT-domains.txt",
"cmd": "cat $WORKSPACE/formatted/$OUTPUT-domains.txt | $GO_PATH/just-resolved | tee $WORKSPACE/formatted/ip-$OUTPUT.txt",
"output_path": "$WORKSPACE/formatted/ip-$OUTPUT.txt",
"cmd": "cat $WORKSPACE/formatted/$OUTPUT-domains.txt | $GO_PATH/just-resolved | tee -a $WORKSPACE/formatted/$OUTPUT-range.txt",
"output_path": "$WORKSPACE/formatted/$OUTPUT-range.txt",
"std_path": "",
},
{
@@ -121,6 +121,12 @@ class StoScan:
"output_path": "$WORKSPACE/stoscan/takeover-$TARGET-subjack.txt",
"std_path": "$WORKSPACE/stoscan/std-takeover-$TARGET-subjack.std"
},
{
"banner": "subzy",
"cmd": "$GO_PATH/subzy -hide_fails -https -concurrency 20 -targets $WORKSPACE/formatted/resolved-$OUTPUT.txt | tee $WORKSPACE/stoscan/takeover-$TARGET-subzy.txt",
"output_path": "$WORKSPACE/stoscan/takeover-$TARGET-subzy.txt",
"std_path": "$WORKSPACE/stoscan/std-takeover-$TARGET-subzy.std"
},
{
"banner": "massdns resolve IP",
"cmd": "cat $WORKSPACE/formatted/resolved-$OUTPUT.txt | $PLUGINS_PATH/massdns/bin/massdns -r $DATA_PATH/resolvers.txt -q -t A -o F -w $WORKSPACE/stoscan/all-dig-info.txt",
@@ -135,6 +141,13 @@ class StoScan:
"output_path": "$WORKSPACE/stoscan/have-cname.txt",
"std_path": "",
},
{
"requirement": "$WORKSPACE/stoscan/all-dig-info.txt",
"banner": "rgf extract Azure",
"cmd": "$GO_PATH/rgf -file $WORKSPACE/stoscan/all-dig-info.txt azure | tee $WORKSPACE/stoscan/azure-sto.txt",
"output_path": "$WORKSPACE/stoscan/azure-sto.txt",
"std_path": "",
},
],
}
@@ -172,6 +185,15 @@ class ParamFinding:
logs = []
commands = {
'general': [
{
"banner": "Arjun param finding",
"cmd": "$ALIAS_PATH/paramfinder -i '[[0]]' -o '$WORKSPACE/params/raw' -s '$WORKSPACE/params/summary-$OUTPUT.txt' -p '$PLUGINS_PATH'",
"output_path": "",
"std_path": "",
"chunk": 5,
"cmd_type": "list",
"resources": "l0|$WORKSPACE/formatted/$OUTPUT-paths.txt",
},
],
}
@@ -245,13 +267,13 @@ class ScreenShot:
{
"requirement": "$WORKSPACE/formatted/http-$OUTPUT.txt",
"banner": "aquatone",
"cmd": f"cat $WORKSPACE/formatted/http-$OUTPUT.txt | $GO_PATH/aquatone -scan-timeout 1000 -threads {threads} -out $WORKSPACE/screenshot/$OUTPUT-aquatone",
"cmd": f"cat $WORKSPACE/formatted/http-$OUTPUT.txt | $GO_PATH/aquatone -screenshot-timeout 50000 -threads {threads} -out $WORKSPACE/screenshot/$OUTPUT-aquatone",
"output_path": "$WORKSPACE/screenshot/$OUTPUT-aquatone/aquatone_report.html",
"std_path": "$WORKSPACE/screenshot/std-$OUTPUT-aquatone.std"
},
{
"banner": "gowitness",
"cmd": f"$GO_PATH/gowitness file -s $WORKSPACE/formatted/http-$OUTPUT.txt -t {threads} --log-level fatal --destination $WORKSPACE/screenshot/raw-gowitness/ --db $WORKSPACE/screenshot/gowitness.db",
"cmd": f"$GO_PATH/gowitness file -s $WORKSPACE/formatted/http-$OUTPUT.txt -t {threads} --timeout 10 --log-level fatal --destination $WORKSPACE/screenshot/raw-gowitness/ --db $WORKSPACE/screenshot/gowitness.db",
"output_path": "$WORKSPACE/screenshot/gowitness.db",
"std_path": "",
},
@@ -294,7 +316,7 @@ class DirbScan:
},
{
"banner": "ffuf dirscan",
"cmd": "$ALIAS_PATH/dirscan -i [[0]] -w '$DATA_PATH/wordlists/content/quick-content-discovery.txt' -o '$WORKSPACE/directory/raw' -p '$GO_PATH' -s '$WORKSPACE/directory'",
"cmd": "$ALIAS_PATH/dirscan -i [[0]] -w '$DATA_PATH/wordlists/content/quick.txt' -o '$WORKSPACE/directory/raw' -p '$GO_PATH' -s '$WORKSPACE/directory'",
"output_path": "",
"std_path": "",
"chunk": 5,
@@ -304,7 +326,7 @@ class DirbScan:
{
"requirement": "$WORKSPACE/formatted/http-$OUTPUT.txt",
"banner": "csv beautify",
"cmd": "cat $WORKSPACE/directory/directory-summary.csv | csvlook --max-column-width 100 | tee $WORKSPACE/directory/beautify-summary.csv",
"cmd": "cat $WORKSPACE/directory/raw/* | csvcut -c 2-6 | csvlook | tee $WORKSPACE/directory/beautify-summary.csv",
"output_path": "",
"std_path": "",
"waiting": "last",
@@ -321,7 +343,7 @@ class PortScan:
"note": "final",
},
{
"path": "$WORKSPACE/portscan/$OUTPUT.html",
"path": "$WORKSPACE/portscan/screenshot-$OUTPUT.html",
"type": "html",
"note": "final",
},
@@ -330,24 +352,19 @@ class PortScan:
"type": "bash",
"note": "final, slack, diff",
},
{
"path": "$WORKSPACE/portscan/$OUTPUT-aquatone/aquatone_report.html",
"type": "html",
"note": "final",
},
{
"path": "$WORKSPACE/portscan/screenshot/$OUTPUT-raw-gowitness.html",
"type": "html",
"note": "final",
"note": "",
},
]
logs = []
commands = {
'general': [
{
"requirement": "$WORKSPACE/probing/ip-$OUTPUT.txt",
"requirement": "$WORKSPACE/formatted/$OUTPUT-range.txt",
"banner": "Masscan 65535 ports",
"cmd": "$ALIAS_PATH/portscan -i $WORKSPACE/probing/ip-$OUTPUT.txt -o '$WORKSPACE/portscan/$OUTPUT' -s '$WORKSPACE/portscan/summary.txt' -p '$PLUGINS_PATH'",
"cmd": "$ALIAS_PATH/portscan -i $WORKSPACE/formatted/$OUTPUT-range.txt -o '$WORKSPACE/portscan/$OUTPUT' -s '$WORKSPACE/portscan/summary.txt' -p '$PLUGINS_PATH'",
"output_path": "$WORKSPACE/portscan/$OUTPUT.csv",
"std_path": "",
"waiting": "first",
@@ -363,12 +380,13 @@ class PortScan:
},
{
"requirement": "$WORKSPACE/portscan/$OUTPUT.csv",
"banner": "CSV beautify",
"banner": "Httprode new port",
"cmd": '''cat $WORKSPACE/portscan/$OUTPUT.csv | awk -F',' '{print $1":"$4}' | httprobe -c 30 | tee $WORKSPACE/portscan/http-$OUTPUT.txt''',
"output_path": "$WORKSPACE/portscan/http-$OUTPUT.txt",
"std_path": "",
},
{
"requirement": "$WORKSPACE/portscan/http-$OUTPUT.txt",
"banner": "aquatone",
"cmd": f"cat $WORKSPACE/portscan/http-$OUTPUT.txt | $GO_PATH/aquatone -screenshot-timeout 50000 -threads {threads} -out $WORKSPACE/portscan/$OUTPUT-aquatone",
"output_path": "$WORKSPACE/portscan/$OUTPUT-aquatone/aquatone_report.html",
lib/workflow/general.py
+37 -80
View File
@@ -6,20 +6,6 @@ import multiprocessing
cpu_cores = multiprocessing.cpu_count()
threads = str(cpu_cores * 3)
class Sample:
''' Sample module '''
reports = [] # point out reports file
logs = [] # point out logs file
# point out commands will run for each mode
commands = {
'general': [], # run any time
'quick': [], # default mode
'slow': [], # will run if specific --slow '<module name>'
'direct': [], # will run in direct mode which is '-i' or '-I' options
}
class SubdomainScanning:
reports = [
{
@@ -33,7 +19,7 @@ class SubdomainScanning:
'general': [
{
"banner": "Amass",
"cmd": "$GO_PATH/amass enum -timeout 5 -active -max-dns-queries 10000 -include-unresolvable -active -dir $WORKSPACE/subdomain/amass-$OUTPUT -d $TARGET -o $WORKSPACE/subdomain/$OUTPUT-amass.txt",
"cmd": "$GO_PATH/amass enum -timeout 10 -active -max-dns-queries 10000 -include-unresolvable -dir $WORKSPACE/subdomain/amass-$OUTPUT -d $TARGET -o $WORKSPACE/subdomain/$OUTPUT-amass.txt",
"output_path": "$WORKSPACE/subdomain/$OUTPUT-amass.txt",
"std_path": "$WORKSPACE/subdomain/std-$TARGET-amass.std"
},
@@ -78,55 +64,6 @@ class SubdomainScanning:
}
class VhostScan:
note = "Pro-only"
reports = [
{
"path": "$WORKSPACE/vhosts/vhost-$OUTPUT.txt",
"type": "bash",
"note": "final, slack",
}
]
logs = []
commands = {
'general': [
{
"banner": "Gobuster Vhost",
"cmd": "$ALIAS_PATH/vhosts -i '[[0]]' -o '$WORKSPACE/vhosts/raw' -s '$WORKSPACE/vhosts/raw-summary-$OUTPUT.txt' -p '$PLUGINS_PATH' -w $DATA_PATH/wordlists/dns/virtual-host-scanning.txt",
"output_path": "",
"std_path": "",
"chunk": 5,
"cmd_type": "list",
"resources": "l0|$WORKSPACE/subdomain/final-$OUTPUT.txt",
"post_run": "clean_vhosts_gobuster",
"cleaned_output": "$WORKSPACE/vhosts/vhosts-$OUTPUT.txt",
},
],
}
class PermutationScan:
note = "Pro-only"
reports = [
{
"path": "$WORKSPACE/permutation/permutation-$OUTPUT.txt",
"type": "bash",
"note": "final",
}
]
logs = []
commands = {
'general': [
{
"banner": "goaltdns",
"cmd": "$GO_PATH/goaltdns -w $DATA_PATH/wordlists/dns/short-permutation.txt -l $WORKSPACE/subdomain/final-$OUTPUT.txt -o $WORKSPACE/permutation/permutation-$OUTPUT.txt",
"output_path": "$WORKSPACE/permutation/permutation-$OUTPUT.txt",
"std_path": "",
},
],
}
class Probing:
reports = [
{
@@ -134,6 +71,11 @@ class Probing:
"type": "bash",
"note": "final, slack, diff",
},
{
"path": "$WORKSPACE/probing/raw-allmassdns.txt",
"type": "bash",
"note": "final",
},
{
"path": "$WORKSPACE/probing/resolved-$OUTPUT.txt",
"type": "bash",
@@ -149,13 +91,12 @@ class Probing:
commands = {
'general': [
{
# this only run if something wrong with custom resolvers
"banner": "massdns resolve IP",
"requirement": "$WORKSPACE/probing/raw-all-$OUTPUT.txt",
"cmd": "cat $WORKSPACE/probing/raw-all-$OUTPUT.txt | $PLUGINS_PATH/massdns/bin/massdns -r $DATA_PATH/resolvers.txt -q -t A -o S -w $WORKSPACE/probing/raw-allmassdns.txt",
"output_path": "$WORKSPACE/probing/raw-allmassdns.txt",
"std_path": "",
"waiting": "last",
"waiting": "first",
"pre_run": "get_subdomains",
"post_run": "clean_massdns",
"cleaned_output": "$WORKSPACE/probing/ip-$OUTPUT.txt",
@@ -163,10 +104,9 @@ class Probing:
{
"banner": "httprobe",
"requirement": "$WORKSPACE/probing/raw-all-$OUTPUT.txt",
"cmd": "cat $WORKSPACE/probing/raw-all-$OUTPUT.txt | $GO_PATH/httprobe -c 100 -t 20000 | tee $WORKSPACE/probing/http-$OUTPUT.txt",
"cmd": "cat $WORKSPACE/probing/raw-all-$OUTPUT.txt | $GO_PATH/httprobe -c 50 -t 20000 | tee $WORKSPACE/probing/http-$OUTPUT.txt",
"output_path": "$WORKSPACE/probing/http-$OUTPUT.txt",
"std_path": "$WORKSPACE/probing/std-http-$OUTPUT.std",
"waiting": "last",
"post_run": "get_domain",
"cleaned_output": "$WORKSPACE/probing/domains-$OUTPUT.txt",
},
@@ -225,7 +165,12 @@ class Fingerprint:
"path": "$WORKSPACE/fingerprint/$OUTPUT-technology.json",
"type": "bash",
"note": "final",
}
},
{
"path": "$WORKSPACE/fingerprint/responses/index",
"type": "bash",
"note": "final",
},
]
logs = []
commands = {
@@ -240,14 +185,20 @@ class Fingerprint:
},
{
"banner": "meg /",
"cmd": "$GO_PATH/meg / $WORKSPACE/probing/http-$OUTPUT.txt $WORKSPACE/fingerprint/responses/ -v -c 100",
"cmd": "$GO_PATH/meg -v -c 100 / $WORKSPACE/probing/http-$OUTPUT.txt $WORKSPACE/fingerprint/responses/",
"output_path": "$WORKSPACE/fingerprint/responses/index",
"std_path": "",
},
{
"banner": "Get success page",
"cmd": "cat $WORKSPACE/fingerprint/responses/index | grep '200 OK' | awk '{print $2}' | tee $WORKSPACE/fingerprint/success-http-$OUTPUT.txt",
"output_path": "$WORKSPACE/fingerprint/success-http-$OUTPUT.txt",
"std_path": "",
},
{
"requirement": "$WORKSPACE/fingerprint/responses/index",
"banner": "rgf extract all",
"cmd": "$GO_PATH/rgf -dir $WORKSPACE/fingerprint/responses/ | tee $WORKSPACE/fingerprint/rgf-all-$OUTPUT.txt",
"cmd": "$GO_PATH/rgf -dirir $WORKSPACE/fingerprint/responses/ | tee $WORKSPACE/fingerprint/rgf-all-$OUTPUT.txt",
"output_path": "$WORKSPACE/fingerprint/rgf-all-$OUTPUT.txt",
"std_path": "",
},
@@ -273,13 +224,13 @@ class ScreenShot:
'general': [
{
"banner": "aquatone",
"cmd": f"cat $WORKSPACE/probing/http-$OUTPUT.txt | $GO_PATH/aquatone -scan-timeout 1000 -threads {threads} -out $WORKSPACE/screenshot/$OUTPUT-aquatone",
"cmd": f"cat $WORKSPACE/fingerprint/success-http-$OUTPUT.txt | $GO_PATH/aquatone -threads {threads} -out $WORKSPACE/screenshot/$OUTPUT-aquatone",
"output_path": "$WORKSPACE/screenshot/$OUTPUT-aquatone/aquatone_report.html",
"std_path": "$WORKSPACE/screenshot/std-$OUTPUT-aquatone.std"
},
{
"banner": "gowitness",
"cmd": f"$GO_PATH/gowitness file -s $WORKSPACE/probing/http-$OUTPUT.txt -t {threads} --log-level fatal --destination $WORKSPACE/screenshot/raw-gowitness/ --db $WORKSPACE/screenshot/gowitness.db",
"cmd": f"$GO_PATH/gowitness file -s $WORKSPACE/fingerprint/success-http-$OUTPUT.txt -t {threads} --log-level fatal --destination $WORKSPACE/screenshot/raw-gowitness/ --db $WORKSPACE/screenshot/gowitness.db",
"output_path": "$WORKSPACE/screenshot/gowitness.db",
"std_path": "",
},
@@ -309,7 +260,7 @@ class StoScan:
},
{
"path": "$WORKSPACE/stoscan/all-dig-info.txt",
"type": "bash"
"type": "final, bash"
},
]
logs = []
@@ -317,7 +268,7 @@ class StoScan:
'general': [
{
"banner": "tko-subs",
"cmd": "$GO_PATH/tko-subs -data $DATA_PATH/providers-data.csv -domains $WORKSPACE/probing/domains-$OUTPUT.txt -output $WORKSPACE/stoscan/takeover-$TARGET-tko-subs.txt",
"cmd": "$GO_PATH/tko-subs -data $DATA_PATH/providers-data.csv -domains $WORKSPACE/probing/resolved-$OUTPUT.txt -output $WORKSPACE/stoscan/takeover-$TARGET-tko-subs.txt",
"output_path": "$WORKSPACE/stoscan/takeover-$TARGET-tko-subs.txt",
"std_path": "$WORKSPACE/stoscan/std-takeover-$TARGET-tko-subs.std",
},
@@ -326,10 +277,9 @@ class StoScan:
"cmd": "$GO_PATH/subjack -v -m -c $DATA_PATH/fingerprints.json -w $WORKSPACE/probing/domains-$OUTPUT.txt -t 100 -timeout 30 -o $WORKSPACE/stoscan/takeover-$TARGET-subjack.txt -ssl",
"output_path": "$WORKSPACE/stoscan/takeover-$TARGET-subjack.txt",
"std_path": "$WORKSPACE/stoscan/std-takeover-$TARGET-subjack.std"
},
{
}, {
"banner": "massdns resolve IP",
"cmd": "cat $WORKSPACE/probing/resolved-$OUTPUT.txt | $PLUGINS_PATH/massdns/bin/massdns -r $PLUGINS_PATH/massdns/lists/resolvers.txt -q -t A -o F -w $WORKSPACE/stoscan/all-dig-info.txt",
"cmd": "cat $WORKSPACE/probing/raw-all-$OUTPUT.txt | $PLUGINS_PATH/massdns/bin/massdns -r $PLUGINS_PATH/massdns/lists/resolvers.txt -q -t A -o F -w $WORKSPACE/stoscan/all-dig-info.txt",
"output_path": "$WORKSPACE/stoscan/all-dig-info.txt",
"std_path": "",
"waiting": "first",
@@ -341,6 +291,13 @@ class StoScan:
"output_path": "$WORKSPACE/stoscan/have-cname.txt",
"std_path": "",
},
{
"requirement": "$WORKSPACE/stoscan/all-dig-info.txt",
"banner": "rgf extract Azure",
"cmd": "$GO_PATH/rgf -file $WORKSPACE/stoscan/all-dig-info.txt azure | tee $WORKSPACE/stoscan/azure-sto.txt",
"output_path": "$WORKSPACE/stoscan/azure-sto.txt",
"std_path": "",
},
],
}
@@ -566,7 +523,7 @@ class DirbScan:
},
{
"banner": "ffuf dirscan",
"cmd": "$ALIAS_PATH/dirscan -i [[0]] -w '$DATA_PATH/wordlists/content/top10000.txt' -o '$WORKSPACE/directory/raw' -p '$GO_PATH' -s '$WORKSPACE/directory'",
"cmd": "$ALIAS_PATH/dirscan -i [[0]] -w '$DATA_PATH/wordlists/content/quick.txt' -o '$WORKSPACE/directory/raw' -p '$GO_PATH' -s '$WORKSPACE/directory'",
"output_path": "",
"std_path": "",
"chunk": 5,
@@ -576,7 +533,7 @@ class DirbScan:
{
"requirement": "$WORKSPACE/probing/http-$OUTPUT.txt",
"banner": "csv beautify",
"cmd": "cat $WORKSPACE/directory/directory-summary.csv | csvlook --max-column-width 100 | tee $WORKSPACE/directory/beautify-summary.csv",
"cmd": "cat $WORKSPACE/directory/raw/* | csvcut -c 2-6 | csvlook | tee $WORKSPACE/directory/beautify-summary.csv",
"output_path": "",
"std_path": "",
"waiting": "last",