diff --git a/Dockerfile b/Dockerfile index 6cdd39e..6e9d351 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,38 +1,21 @@ -FROM debian:latest - -RUN apt-get update && \ - apt-get -qq install \ - npm \ - locales \ - git \ - sudo \ - wget \ - python3-pip \ - python-pip \ - curl \ - libcurl4-openssl-dev \ - bsdmainutils \ - xsltproc \ - build-essential - -# Set the locale -RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ - dpkg-reconfigure --frontend=noninteractive locales && \ - update-locale LANG=en_US.UTF-8 - -ENV LANG en_US.UTF-8 - -RUN cp -av /usr/bin/pip2 /usr/bin/pip2.7 && \ - pip install setuptools && \ - pip3 install setuptools && \ - pip install wheel && \ - pip3 install wheel - -COPY . /home/Osmedeus +FROM debian:buster-20191014-slim +ARG OSMEDEUS_VERSION=v.2.1 +RUN sed -i 's/main/main contrib non-free/' /etc/apt/sources.list WORKDIR /home/Osmedeus - -RUN ./install.sh && \ +ENV LANG="en_US.UTF-8" \ + LANGUAGE="en_US:en" \ + LC_ALL="en_US.UTF-8" +RUN apt-get update && \ + apt-get -yu dist-upgrade && \ + apt-get -qq install npm && \ + apt-get -qq install locales git sudo wget python3-pip python-pip curl libcurl4-openssl-dev bsdmainutils xsltproc && \ + sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ + locale-gen && \ + cp -av /usr/bin/pip2 /usr/bin/pip2.7 && \ + git clone --depth 1 https://github.com/j3ssie/Osmedeus -b $OSMEDEUS_VERSION . && \ + ./install.sh && \ + go get -u github.com/tomnomnom/unfurl && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* - -ENTRYPOINT ["python3", "server/manage.py", "runserver", "0.0.0.0:8000"] \ No newline at end of file +EXPOSE 8000 +CMD ["./osmedeus.py"] \ No newline at end of file diff --git a/README.md b/README.md index 748ab02..35fbbf4 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,10 @@ cd Osmedeus This install only focus on Kali linux, check more install on [Usage page](https://j3ssie.github.io/Osmedeus/installation/) +## Using Docker + +Check out [docker-osmedeus](https://github.com/mablanco/docker-osmedeus) by [mabnavarrete](https://twitter.com/mabnavarrete) + ## How to use If you have no idea what are you doing just type the command below or check out the [Advanced Usage](https://j3ssie.github.io/Osmedeus/advanced/) @@ -64,10 +68,6 @@ If you love my tool and would like to support my work. [Become a backer](https:/ ## Demo [![asciicast](https://asciinema.org/a/266599.svg)](https://asciinema.org/a/266599) -![Reports](imgs/osmedeus-report.png) - -![Slack Noti](imgs/slack_noti.png) - ![Web UI 1](imgs/ui-1.png) ![Web UI 3](imgs/ui-3.png) @@ -89,8 +89,6 @@ If you love my tool and would like to support my work. [Become a backer](https:/ ./osmedeus.py -m portscan -i "1.2.3.4/24" ./osmedeus.py -m "portscan,vulnscan" -i "1.2.3.4/24" -w result_folder -./osmedeus.py -m "git" -i 'repo:https://github.com/foo/bar' -./osmedeus.py -m "git" -i 'user:sample' # direct list mode examples ./osmedeus.py -m portscan -I list_of_targets.txt @@ -98,14 +96,19 @@ If you love my tool and would like to support my work. [Become a backer](https:/ ./osmedeus.py -m screen -I list_of_targets.txt -w result_folder # report mode -./osemdeus.py -t example.com --report list -./osemdeus.py -t example.com --report export -./osemdeus.py -t example.com --report sum -./osemdeus.py -t example.com --report short -./osemdeus.py -t example.com --report full +./osmedeus.py -t example.com --report list +./osmedeus.py -t example.com --report export +./osmedeus.py -t example.com --report sum +./osmedeus.py -t example.com --report short +./osmedeus.py -t example.com --report full ``` +## What to do next after do recon? + +Check out my scanner [Jaeles](https://github.com/jaeles-project/jaeles). + + ## Disclaimer Most of this tool done by the authors of the tool that list in [CREDITS.md](https://github.com/j3ssie/Osmedeus/blob/master/CREDITS.md). diff --git a/imgs/osmedeus-architecture.png b/imgs/osmedeus-architecture.png index 9892e44..16cc023 100644 Binary files a/imgs/osmedeus-architecture.png and b/imgs/osmedeus-architecture.png differ diff --git a/install.sh b/install.sh index e483939..f395f89 100755 --- a/install.sh +++ b/install.sh @@ -23,12 +23,13 @@ install_banner "git, nmap, masscan, chromium, npm, golang" [ -x "$(command -v go)" ] || sudo $PACKGE_MANAGER install golang -y 2>/dev/null [ -x "$(command -v make)" ] || sudo $PACKGE_MANAGER install build-essential -y 2>/dev/null [ -x "$(command -v csvlook)" ] || sudo $PACKGE_MANAGER install csvkit -y 2>/dev/null -[ -x "$(command -v ripgrep)" ] || sudo $PACKGE_MANAGER install ripgrep -y 2>/dev/null +[ -x "$(command -v rg)" ] || sudo $PACKGE_MANAGER install ripgrep -y 2>/dev/null [ -x "$(command -v unzip)" ] || sudo $PACKGE_MANAGER install unzip -y 2>/dev/null [ -x "$(command -v chromium-browser)" ] || sudo $PACKGE_MANAGER install chromium-browser -y 2>/dev/null +[ -x "$(command -v xsltproc)" ] || sudo $PACKGE_MANAGER install xsltproc -y 2>/dev/null +[ -x "$(command -v make)" ] || sudo $PACKGE_MANAGER install build-essential -y 2>/dev/null [ -x "$(command -v pip)" ] || sudo $PACKGE_MANAGER install python-pip -y 2>/dev/null [ -x "$(command -v pip3)" ] || sudo $PACKGE_MANAGER install python3-pip -y 2>/dev/null -[ -x "$(command -v xsltproc)" ] || sudo $PACKGE_MANAGER install xsltproc -y 2>/dev/null pip install setuptools 2>/dev/null pip3 install setuptools 2>/dev/null @@ -128,6 +129,7 @@ python3 server/manage.py migrate python3 server/manage.py makemigrations api python3 server/manage.py migrate api python3 scripts/init.py +python3 scripts/reload.py ### adding gopath if GOPATH not in default shellrc if ! grep -Fxq "GOPATH" "$DEFAULT_SHELL"; then @@ -148,8 +150,6 @@ GO_BIN=$(which go) ## # Install go stuff ## -install_banner "amass" -$GO_BIN get -u github.com/OWASP/Amass/... install_banner "subfinder" $GO_BIN get -u github.com/subfinder/subfinder install_banner "gobuster" @@ -188,9 +188,15 @@ install_banner "ffuf" $GO_BIN get -u github.com/ffuf/ffuf install_banner "rgf" $GO_BIN get -u github.com/j3ssie/rgf +install_banner "jaeles" +$GO_BIN get -u github.com/jaeles-project/jaeles install_banner "go cli-utils" +$GO_BIN get -u github.com/j3ssie/go-auxs/getIP $GO_BIN get -u github.com/j3ssie/go-auxs/just-resolved -$GO_BIN get -u github.com/j3ssie/go-auxs/cinfo + +install_banner "amass" +export GO111MODULE=on +go get -u github.com/OWASP/Amass/v3/... cp $GO_DIR/* "$PLUGINS_PATH/go/" 2>/dev/null # install_banner "observatory" @@ -221,7 +227,7 @@ chmod +x $PLUGINS_PATH/findomain # Install python stuff ## -install_banner "truffleHog, wfuzz" +install_banner "truffleHog" pip install truffleHog cd $PLUGINS_PATH @@ -238,12 +244,9 @@ install_banner "Metabigor" git clone https://github.com/j3ssie/Metabigor 2>/dev/null pip3 install -r Metabigor/requirements.txt -install_banner "bass" -git clone https://github.com/Abss0x7tbh/bass 2>/dev/null -pip3 install -r bass/requirements.txt - install_banner "dirsearch" git clone https://github.com/maurosoria/dirsearch 2>/dev/null + install_banner "Arjun" git clone https://github.com/s0md3v/Arjun 2>/dev/null diff --git a/lib/alias/dirscan b/lib/alias/dirscan index fa44bf6..40d1ede 100755 --- a/lib/alias/dirscan +++ b/lib/alias/dirscan @@ -38,10 +38,10 @@ def routine(args): # init the summary if not utils.not_empty_file(f'{final_summary}'): - head = "input,position,status_code,content_length,content_words\n" + head = "FUZZ,url,redirectlocation,position,status_code,content_length,content_words,content_lines\n" utils.just_write(final_summary, head) - sep = f"\n{target},,,,\n" + sep = f"{target},,,,\n" utils.just_append(final_summary, sep + "\n".join(data)) diff --git a/lib/alias/format_input b/lib/alias/format_input index d44b6c1..3cdcff9 100755 --- a/lib/alias/format_input +++ b/lib/alias/format_input @@ -3,6 +3,7 @@ import os import sys import time import argparse +import re import urllib.parse sys.path.append(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) @@ -10,6 +11,16 @@ from core import utils # portscan -i /tmp/target.txt -o '$WORKSPACE/portscan/$OUTPUT' -s '$WORKSPACE/portscan/summary.txt' -p '$PLUGINS_PATH' +def grep_the_IP(data): + cidr_regex = "((\d){1,3}\.){3}(\d){1,3}(\/(\d){1,3})?" + ips = [] + p = re.compile(cidr_regex) + for m in p.finditer(data): + ips.append(m.group()) + if ips: + return ips[0] + return ips + def routine(args): target = args.input output = args.output @@ -23,7 +34,13 @@ def routine(args): f'cat {target} | unfurl -u format https://%d:%P >> {output}-port.txt') os.system(f'cat {target} | unfurl -u format %d >> {output}-domains.txt') os.system(f'cat {target} | unfurl -u format %r >> {output}-root.txt') - time.sleep(0.3) + + ips = [] + data = utils.just_read(target, get_list=True) + for line in data: + ips.append(grep_the_IP(line)) + utils.just_write(f"{output}-range.txt", "\n".join(ips)) + time.sleep(0.2) parser = argparse.ArgumentParser(description="VulnScan alias") diff --git a/lib/alias/paramfinder b/lib/alias/paramfinder new file mode 100755 index 0000000..ffd299b --- /dev/null +++ b/lib/alias/paramfinder @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +import os +import sys +import time +import argparse +import urllib.parse +sys.path.append(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) + +from core import utils + + +# $ALIAS_PATH/paramfinding -i '[[0]]' -o '$WORKSPACE/params/raw' -s '$WORKSPACE/params/summary-$OUTPUT.txt' -p '$PLUGINS_PATH' +def routine(args): + target = args.input + output = args.output + strip_target = utils.strip_slash(target) + plugin = args.plugin + summary = args.summary + + cmd = f'python3 {plugin}/Arjun/arjun.py -t 40 -f {plugin}/Arjun/db/params.txt --get -u {target} > {output}/{strip_target}.txt' + utils.print_info(f"Execute: {cmd}") + os.system(cmd) + contents = utils.just_read(f'{output}/{strip_target}.txt') + if 'parameter found:' in contents or 'potential parameter:' in contents: + params = [] + for line in contents.split("\n"): + if 'Valid parameter found: ' in line: + params.append(line.split(': ')[1]) + if 'potential parameter: ' in line: + params.append(line.split(': ')[1]) + data = f"{target} | params: " + ", ".join(list(set(params))) + "\n" + # append to summary + utils.just_append(summary, data) + time.sleep(0.3) + + +parser = argparse.ArgumentParser(description="LinkFinder alias") +parser.add_argument('-i', '--input', action='store', dest='input', help='input') +parser.add_argument('-o', '--output', action='store', dest='output', help='output') +parser.add_argument('-p', '--plugin', action='store', dest='plugin', help='plugin') +parser.add_argument('-s', '--summary', action='store', + dest='summary', help='summary') +args = parser.parse_args() +routine(args) + diff --git a/lib/client/helpers.py b/lib/client/helpers.py index 79808af..272a629 100644 --- a/lib/client/helpers.py +++ b/lib/client/helpers.py @@ -88,8 +88,8 @@ python3 osmedeus.py -w sample2 -m dirb -I /tmp/list_of_hosts.txt -s, --slow "subdomain" Only running slow mode in subdomain module ---noslack Disable slack notification ---nomonitor Disable monitor new assets +--slack Enable slack notification +--monitor Enable monitor new assets --debug Just for debug purpose '''.format(G, GR, B)) sys.exit(0) @@ -118,15 +118,15 @@ gitscan, cors, ipspace {1}[{0}Report Usage{1}]{1} =================== -./osemdeus.py --report -t [-m ] +./osmedeus.py --report -t [-m ] {1}[{0}Example Commands{1}]{1} =================== -./osemdeus.py -t example.com --report list -./osemdeus.py -t example.com --report sum -./osemdeus.py -t example.com --report path -./osemdeus.py -t example.com --report short -./osemdeus.py -t example.com -m subdomain --report short -./osemdeus.py -t example.com -m subdomain, portscan --report short -./osemdeus.py -t example.com -m subdomain, portscan --report full +./osmedeus.py -t example.com --report list +./osmedeus.py -t example.com --report sum +./osmedeus.py -t example.com --report path +./osmedeus.py -t example.com --report short +./osmedeus.py -t example.com -m subdomain --report short +./osmedeus.py -t example.com -m subdomain, portscan --report short +./osmedeus.py -t example.com -m subdomain, portscan --report full '''.format(G, GR, B)) diff --git a/lib/core/utils.py b/lib/core/utils.py index 2039d3d..7981046 100644 --- a/lib/core/utils.py +++ b/lib/core/utils.py @@ -126,7 +126,7 @@ def print_elapsed(options): def print_debug(text, options=None): if not options: return - if options.get('DEBUG'): + if options.get('DEBUG', False) or options.get('debug', False): print(G + "#" * 20 + GR) print(text) print("#" * 20) diff --git a/lib/mode/direct.py b/lib/mode/direct.py index 1f640e1..38244ec 100644 --- a/lib/mode/direct.py +++ b/lib/mode/direct.py @@ -32,12 +32,8 @@ def handle(options): if utils.loop_grep(modules, 'sub'): subdomain.SubdomainScanning(options) - # vhosts.VhostScan(options) - # permutation.PermutationScan(options) probing.Probing(options) fingerprint.Fingerprint(options) - # stoscan.StoScan(options) - # screenshot.ScreenShot(options) if utils.loop_grep(modules, 'screen'): screenshot.ScreenShot(options) diff --git a/lib/mode/general.py b/lib/mode/general.py index 9b379e7..5728f8c 100644 --- a/lib/mode/general.py +++ b/lib/mode/general.py @@ -4,8 +4,6 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) from lib.core import utils from modules import subdomain -from modules import vhosts -from modules import permutation from modules import probing from modules import formatting from modules import fingerprint @@ -33,8 +31,6 @@ def handle(options): def single_handle(options): subdomain.SubdomainScanning(options) - # vhosts.VhostScan(options) - # permutation.PermutationScan(options) probing.Probing(options) formatting.Formatting(options) corscan.CORScan(options) diff --git a/lib/mode/routine.py b/lib/mode/routine.py index 7aade58..b5b415d 100644 --- a/lib/mode/routine.py +++ b/lib/mode/routine.py @@ -18,7 +18,8 @@ def routine_handle(options): utils.print_target(options.get('TARGET')) # move last result to monitor path and add options to compare - if options.get('MONITOR'): + # only do monitor on general mode + if options.get('MONITOR') and options['MODE'] == "general": options['COMPARE_PATH'] = backup.init_backup(options) utils.print_good("Created compare workspaces: {0}".format( options['COMPARE_PATH'])) diff --git a/lib/workflow/direct.py b/lib/workflow/direct.py index c220559..bfeba7f 100644 --- a/lib/workflow/direct.py +++ b/lib/workflow/direct.py @@ -5,7 +5,6 @@ cpu_cores = multiprocessing.cpu_count() threads = str(cpu_cores * 2) - class SubdomainScanning: reports = [ { @@ -19,7 +18,7 @@ class SubdomainScanning: 'general': [ { "banner": "Amass", - "cmd": "$GO_PATH/amass enum -timeout 5 -active -max-dns-queries 10000 -include-unresolvable -active -dir $WORKSPACE/subdomain/amass-$OUTPUT -d $TARGET -o $WORKSPACE/subdomain/$OUTPUT-amass.txt", + "cmd": "$GO_PATH/amass enum -timeout 10 -active -max-dns-queries 10000 -include-unresolvable -active -dir $WORKSPACE/subdomain/amass-$OUTPUT -d $TARGET -o $WORKSPACE/subdomain/$OUTPUT-amass.txt", "output_path": "$WORKSPACE/subdomain/$OUTPUT-amass.txt", "std_path": "$WORKSPACE/subdomain/std-$TARGET-amass.std" }, @@ -120,6 +119,11 @@ class Probing: "type": "bash", "note": "final, slack, diff", }, + { + "path": "$WORKSPACE/probing/raw-allmassdns.txt", + "type": "bash", + "note": "final", + }, { "path": "$WORKSPACE/probing/resolved-$OUTPUT.txt", "type": "bash", @@ -233,7 +237,7 @@ class Fingerprint: { "requirement": "$WORKSPACE/fingerprint/responses/index", "banner": "rgf extract all", - "cmd": "$GO_PATH/rgf -d $WORKSPACE/fingerprint/responses/ | tee $WORKSPACE/fingerprint/rgf-all-$OUTPUT.txt", + "cmd": "$GO_PATH/rgf -dir $WORKSPACE/fingerprint/responses/ | tee $WORKSPACE/fingerprint/rgf-all-$OUTPUT.txt", "output_path": "$WORKSPACE/fingerprint/rgf-all-$OUTPUT.txt", "std_path": "", }, @@ -421,15 +425,20 @@ class PortScan: "type": "html", "note": "final", }, + { + "path": "$WORKSPACE/portscan/screenshot/$OUTPUT-raw-gowitness.html", + "type": "html", + "note": "", + }, ] logs = [] commands = { 'general': [ { - "requirement": "$WORKSPACE/probing/ip-$OUTPUT.txt", + # "requirement": "$WORKSPACE/formatted/ip-$OUTPUT.txt", "banner": "Masscan 65535 ports", - "cmd": "$ALIAS_PATH/portscan -i $WORKSPACE/probing/ip-$OUTPUT.txt -o '$WORKSPACE/portscan/$OUTPUT' -s '$WORKSPACE/portscan/summary.txt' -p '$PLUGINS_PATH'", - "output_path": "$WORKSPACE/portscan/$OUTPUT.csv", + "cmd": "$ALIAS_PATH/portscan -i $TARGET -o '$WORKSPACE/portscan/$OUTPUT' -s '$WORKSPACE/portscan/summary.txt' -p '$PLUGINS_PATH'", + "output_path": "$WORKSPACE/portscan/$OUTPUT.xml", "std_path": "", "waiting": "first", }, @@ -444,18 +453,32 @@ class PortScan: }, { "requirement": "$WORKSPACE/portscan/$OUTPUT.csv", - "banner": "CSV beautify", + "banner": "Httprode new port", "cmd": '''cat $WORKSPACE/portscan/$OUTPUT.csv | awk -F',' '{print $1":"$4}' | httprobe -c 30 | tee $WORKSPACE/portscan/http-$OUTPUT.txt''', "output_path": "$WORKSPACE/portscan/http-$OUTPUT.txt", "std_path": "", }, { + "requirement": "$WORKSPACE/portscan/http-$OUTPUT.txt", "banner": "aquatone", "cmd": f"cat $WORKSPACE/portscan/http-$OUTPUT.txt | $GO_PATH/aquatone -screenshot-timeout 50000 -threads {threads} -out $WORKSPACE/portscan/$OUTPUT-aquatone", "output_path": "$WORKSPACE/portscan/$OUTPUT-aquatone/aquatone_report.html", "std_path": "$WORKSPACE/portscan/std-$OUTPUT-aquatone.std", "waiting": "last", }, + + # { + # "requirement": "$WORKSPACE/portscan/$OUTPUT.csv", + # "banner": "Screenshot on ports found", + # "cmd": "$GO_PATH/gowitness file -s $WORKSPACE/portscan/scheme-$OUTPUT.txt -t 30 --log-level fatal --destination $WORKSPACE/portscan/screenshot/raw-gowitness/ --db $WORKSPACE/portscan/screenshot/gowitness.db", + # "output_path": "$WORKSPACE/portscan/screenshot/gowitness.db", + # "std_path": "", + # "post_run": "clean_gowitness", + # "pre_run": "get_scheme", + # "cleaned_output": "$WORKSPACE/portscan/screenshot-$OUTPUT.html", + # "waiting": "last", + # }, + ], } @@ -525,7 +548,7 @@ class IPSpace: 'general': [ { "banner": "Metabigor IP Lookup", - "cmd": "$PLUGINS_PATH/Metabigor/metabigor.py -m ip -t $TARGET -o $WORKSPACE/ipspace/range-$OUTPUT.txt", + "cmd": "PLUGINS_PATH/Metabigor/metabigor.py -m ip -t $TARGET -o $WORKSPACE/ipspace/range-$OUTPUT.txt", "output_path": "$WORKSPACE/ipspace/range-$OUTPUT.txt", "std_path": "", "post_run": "get_amass", diff --git a/lib/workflow/direct_list.py b/lib/workflow/direct_list.py index 8519b4f..9f3e3ef 100644 --- a/lib/workflow/direct_list.py +++ b/lib/workflow/direct_list.py @@ -11,7 +11,7 @@ class Formatting: "type": "bash" }, { - "path": "$WORKSPACE/formatted/ip-$OUTPUT.txt", + "path": "$WORKSPACE/formatted/$OUTPUT-range.txt", "type": "bash" } ] @@ -29,8 +29,8 @@ class Formatting: { "banner": "Resolve IP", "requirement": "$WORKSPACE/formatted/$OUTPUT-domains.txt", - "cmd": "cat $WORKSPACE/formatted/$OUTPUT-domains.txt | $GO_PATH/just-resolved | tee $WORKSPACE/formatted/ip-$OUTPUT.txt", - "output_path": "$WORKSPACE/formatted/ip-$OUTPUT.txt", + "cmd": "cat $WORKSPACE/formatted/$OUTPUT-domains.txt | $GO_PATH/just-resolved | tee -a $WORKSPACE/formatted/$OUTPUT-range.txt", + "output_path": "$WORKSPACE/formatted/$OUTPUT-range.txt", "std_path": "", }, { @@ -121,6 +121,12 @@ class StoScan: "output_path": "$WORKSPACE/stoscan/takeover-$TARGET-subjack.txt", "std_path": "$WORKSPACE/stoscan/std-takeover-$TARGET-subjack.std" }, + { + "banner": "subzy", + "cmd": "$GO_PATH/subzy -hide_fails -https -concurrency 20 -targets $WORKSPACE/formatted/resolved-$OUTPUT.txt | tee $WORKSPACE/stoscan/takeover-$TARGET-subzy.txt", + "output_path": "$WORKSPACE/stoscan/takeover-$TARGET-subzy.txt", + "std_path": "$WORKSPACE/stoscan/std-takeover-$TARGET-subzy.std" + }, { "banner": "massdns resolve IP", "cmd": "cat $WORKSPACE/formatted/resolved-$OUTPUT.txt | $PLUGINS_PATH/massdns/bin/massdns -r $DATA_PATH/resolvers.txt -q -t A -o F -w $WORKSPACE/stoscan/all-dig-info.txt", @@ -135,6 +141,13 @@ class StoScan: "output_path": "$WORKSPACE/stoscan/have-cname.txt", "std_path": "", }, + { + "requirement": "$WORKSPACE/stoscan/all-dig-info.txt", + "banner": "rgf extract Azure", + "cmd": "$GO_PATH/rgf -file $WORKSPACE/stoscan/all-dig-info.txt azure | tee $WORKSPACE/stoscan/azure-sto.txt", + "output_path": "$WORKSPACE/stoscan/azure-sto.txt", + "std_path": "", + }, ], } @@ -172,6 +185,15 @@ class ParamFinding: logs = [] commands = { 'general': [ + { + "banner": "Arjun param finding", + "cmd": "$ALIAS_PATH/paramfinder -i '[[0]]' -o '$WORKSPACE/params/raw' -s '$WORKSPACE/params/summary-$OUTPUT.txt' -p '$PLUGINS_PATH'", + "output_path": "", + "std_path": "", + "chunk": 5, + "cmd_type": "list", + "resources": "l0|$WORKSPACE/formatted/$OUTPUT-paths.txt", + }, ], } @@ -245,13 +267,13 @@ class ScreenShot: { "requirement": "$WORKSPACE/formatted/http-$OUTPUT.txt", "banner": "aquatone", - "cmd": f"cat $WORKSPACE/formatted/http-$OUTPUT.txt | $GO_PATH/aquatone -scan-timeout 1000 -threads {threads} -out $WORKSPACE/screenshot/$OUTPUT-aquatone", + "cmd": f"cat $WORKSPACE/formatted/http-$OUTPUT.txt | $GO_PATH/aquatone -screenshot-timeout 50000 -threads {threads} -out $WORKSPACE/screenshot/$OUTPUT-aquatone", "output_path": "$WORKSPACE/screenshot/$OUTPUT-aquatone/aquatone_report.html", "std_path": "$WORKSPACE/screenshot/std-$OUTPUT-aquatone.std" }, { "banner": "gowitness", - "cmd": f"$GO_PATH/gowitness file -s $WORKSPACE/formatted/http-$OUTPUT.txt -t {threads} --log-level fatal --destination $WORKSPACE/screenshot/raw-gowitness/ --db $WORKSPACE/screenshot/gowitness.db", + "cmd": f"$GO_PATH/gowitness file -s $WORKSPACE/formatted/http-$OUTPUT.txt -t {threads} --timeout 10 --log-level fatal --destination $WORKSPACE/screenshot/raw-gowitness/ --db $WORKSPACE/screenshot/gowitness.db", "output_path": "$WORKSPACE/screenshot/gowitness.db", "std_path": "", }, @@ -294,7 +316,7 @@ class DirbScan: }, { "banner": "ffuf dirscan", - "cmd": "$ALIAS_PATH/dirscan -i [[0]] -w '$DATA_PATH/wordlists/content/quick-content-discovery.txt' -o '$WORKSPACE/directory/raw' -p '$GO_PATH' -s '$WORKSPACE/directory'", + "cmd": "$ALIAS_PATH/dirscan -i [[0]] -w '$DATA_PATH/wordlists/content/quick.txt' -o '$WORKSPACE/directory/raw' -p '$GO_PATH' -s '$WORKSPACE/directory'", "output_path": "", "std_path": "", "chunk": 5, @@ -304,7 +326,7 @@ class DirbScan: { "requirement": "$WORKSPACE/formatted/http-$OUTPUT.txt", "banner": "csv beautify", - "cmd": "cat $WORKSPACE/directory/directory-summary.csv | csvlook --max-column-width 100 | tee $WORKSPACE/directory/beautify-summary.csv", + "cmd": "cat $WORKSPACE/directory/raw/* | csvcut -c 2-6 | csvlook | tee $WORKSPACE/directory/beautify-summary.csv", "output_path": "", "std_path": "", "waiting": "last", @@ -321,7 +343,7 @@ class PortScan: "note": "final", }, { - "path": "$WORKSPACE/portscan/$OUTPUT.html", + "path": "$WORKSPACE/portscan/screenshot-$OUTPUT.html", "type": "html", "note": "final", }, @@ -330,24 +352,19 @@ class PortScan: "type": "bash", "note": "final, slack, diff", }, - { - "path": "$WORKSPACE/portscan/$OUTPUT-aquatone/aquatone_report.html", - "type": "html", - "note": "final", - }, { "path": "$WORKSPACE/portscan/screenshot/$OUTPUT-raw-gowitness.html", "type": "html", - "note": "final", + "note": "", }, ] logs = [] commands = { 'general': [ { - "requirement": "$WORKSPACE/probing/ip-$OUTPUT.txt", + "requirement": "$WORKSPACE/formatted/$OUTPUT-range.txt", "banner": "Masscan 65535 ports", - "cmd": "$ALIAS_PATH/portscan -i $WORKSPACE/probing/ip-$OUTPUT.txt -o '$WORKSPACE/portscan/$OUTPUT' -s '$WORKSPACE/portscan/summary.txt' -p '$PLUGINS_PATH'", + "cmd": "$ALIAS_PATH/portscan -i $WORKSPACE/formatted/$OUTPUT-range.txt -o '$WORKSPACE/portscan/$OUTPUT' -s '$WORKSPACE/portscan/summary.txt' -p '$PLUGINS_PATH'", "output_path": "$WORKSPACE/portscan/$OUTPUT.csv", "std_path": "", "waiting": "first", @@ -363,12 +380,13 @@ class PortScan: }, { "requirement": "$WORKSPACE/portscan/$OUTPUT.csv", - "banner": "CSV beautify", + "banner": "Httprode new port", "cmd": '''cat $WORKSPACE/portscan/$OUTPUT.csv | awk -F',' '{print $1":"$4}' | httprobe -c 30 | tee $WORKSPACE/portscan/http-$OUTPUT.txt''', "output_path": "$WORKSPACE/portscan/http-$OUTPUT.txt", "std_path": "", }, { + "requirement": "$WORKSPACE/portscan/http-$OUTPUT.txt", "banner": "aquatone", "cmd": f"cat $WORKSPACE/portscan/http-$OUTPUT.txt | $GO_PATH/aquatone -screenshot-timeout 50000 -threads {threads} -out $WORKSPACE/portscan/$OUTPUT-aquatone", "output_path": "$WORKSPACE/portscan/$OUTPUT-aquatone/aquatone_report.html", diff --git a/lib/workflow/general.py b/lib/workflow/general.py index 4c40bf5..99fb214 100644 --- a/lib/workflow/general.py +++ b/lib/workflow/general.py @@ -6,20 +6,6 @@ import multiprocessing cpu_cores = multiprocessing.cpu_count() threads = str(cpu_cores * 3) - -class Sample: - ''' Sample module ''' - reports = [] # point out reports file - logs = [] # point out logs file - # point out commands will run for each mode - commands = { - 'general': [], # run any time - 'quick': [], # default mode - 'slow': [], # will run if specific --slow '' - 'direct': [], # will run in direct mode which is '-i' or '-I' options - } - - class SubdomainScanning: reports = [ { @@ -33,7 +19,7 @@ class SubdomainScanning: 'general': [ { "banner": "Amass", - "cmd": "$GO_PATH/amass enum -timeout 5 -active -max-dns-queries 10000 -include-unresolvable -active -dir $WORKSPACE/subdomain/amass-$OUTPUT -d $TARGET -o $WORKSPACE/subdomain/$OUTPUT-amass.txt", + "cmd": "$GO_PATH/amass enum -timeout 10 -active -max-dns-queries 10000 -include-unresolvable -dir $WORKSPACE/subdomain/amass-$OUTPUT -d $TARGET -o $WORKSPACE/subdomain/$OUTPUT-amass.txt", "output_path": "$WORKSPACE/subdomain/$OUTPUT-amass.txt", "std_path": "$WORKSPACE/subdomain/std-$TARGET-amass.std" }, @@ -78,55 +64,6 @@ class SubdomainScanning: } -class VhostScan: - note = "Pro-only" - reports = [ - { - "path": "$WORKSPACE/vhosts/vhost-$OUTPUT.txt", - "type": "bash", - "note": "final, slack", - } - ] - logs = [] - commands = { - 'general': [ - { - "banner": "Gobuster Vhost", - "cmd": "$ALIAS_PATH/vhosts -i '[[0]]' -o '$WORKSPACE/vhosts/raw' -s '$WORKSPACE/vhosts/raw-summary-$OUTPUT.txt' -p '$PLUGINS_PATH' -w $DATA_PATH/wordlists/dns/virtual-host-scanning.txt", - "output_path": "", - "std_path": "", - "chunk": 5, - "cmd_type": "list", - "resources": "l0|$WORKSPACE/subdomain/final-$OUTPUT.txt", - "post_run": "clean_vhosts_gobuster", - "cleaned_output": "$WORKSPACE/vhosts/vhosts-$OUTPUT.txt", - }, - ], - } - - -class PermutationScan: - note = "Pro-only" - reports = [ - { - "path": "$WORKSPACE/permutation/permutation-$OUTPUT.txt", - "type": "bash", - "note": "final", - } - ] - logs = [] - commands = { - 'general': [ - { - "banner": "goaltdns", - "cmd": "$GO_PATH/goaltdns -w $DATA_PATH/wordlists/dns/short-permutation.txt -l $WORKSPACE/subdomain/final-$OUTPUT.txt -o $WORKSPACE/permutation/permutation-$OUTPUT.txt", - "output_path": "$WORKSPACE/permutation/permutation-$OUTPUT.txt", - "std_path": "", - }, - ], - } - - class Probing: reports = [ { @@ -134,6 +71,11 @@ class Probing: "type": "bash", "note": "final, slack, diff", }, + { + "path": "$WORKSPACE/probing/raw-allmassdns.txt", + "type": "bash", + "note": "final", + }, { "path": "$WORKSPACE/probing/resolved-$OUTPUT.txt", "type": "bash", @@ -149,13 +91,12 @@ class Probing: commands = { 'general': [ { - # this only run if something wrong with custom resolvers "banner": "massdns resolve IP", "requirement": "$WORKSPACE/probing/raw-all-$OUTPUT.txt", "cmd": "cat $WORKSPACE/probing/raw-all-$OUTPUT.txt | $PLUGINS_PATH/massdns/bin/massdns -r $DATA_PATH/resolvers.txt -q -t A -o S -w $WORKSPACE/probing/raw-allmassdns.txt", "output_path": "$WORKSPACE/probing/raw-allmassdns.txt", "std_path": "", - "waiting": "last", + "waiting": "first", "pre_run": "get_subdomains", "post_run": "clean_massdns", "cleaned_output": "$WORKSPACE/probing/ip-$OUTPUT.txt", @@ -163,10 +104,9 @@ class Probing: { "banner": "httprobe", "requirement": "$WORKSPACE/probing/raw-all-$OUTPUT.txt", - "cmd": "cat $WORKSPACE/probing/raw-all-$OUTPUT.txt | $GO_PATH/httprobe -c 100 -t 20000 | tee $WORKSPACE/probing/http-$OUTPUT.txt", + "cmd": "cat $WORKSPACE/probing/raw-all-$OUTPUT.txt | $GO_PATH/httprobe -c 50 -t 20000 | tee $WORKSPACE/probing/http-$OUTPUT.txt", "output_path": "$WORKSPACE/probing/http-$OUTPUT.txt", "std_path": "$WORKSPACE/probing/std-http-$OUTPUT.std", - "waiting": "last", "post_run": "get_domain", "cleaned_output": "$WORKSPACE/probing/domains-$OUTPUT.txt", }, @@ -225,7 +165,12 @@ class Fingerprint: "path": "$WORKSPACE/fingerprint/$OUTPUT-technology.json", "type": "bash", "note": "final", - } + }, + { + "path": "$WORKSPACE/fingerprint/responses/index", + "type": "bash", + "note": "final", + }, ] logs = [] commands = { @@ -240,14 +185,20 @@ class Fingerprint: }, { "banner": "meg /", - "cmd": "$GO_PATH/meg / $WORKSPACE/probing/http-$OUTPUT.txt $WORKSPACE/fingerprint/responses/ -v -c 100", + "cmd": "$GO_PATH/meg -v -c 100 / $WORKSPACE/probing/http-$OUTPUT.txt $WORKSPACE/fingerprint/responses/", "output_path": "$WORKSPACE/fingerprint/responses/index", "std_path": "", }, + { + "banner": "Get success page", + "cmd": "cat $WORKSPACE/fingerprint/responses/index | grep '200 OK' | awk '{print $2}' | tee $WORKSPACE/fingerprint/success-http-$OUTPUT.txt", + "output_path": "$WORKSPACE/fingerprint/success-http-$OUTPUT.txt", + "std_path": "", + }, { "requirement": "$WORKSPACE/fingerprint/responses/index", "banner": "rgf extract all", - "cmd": "$GO_PATH/rgf -dir $WORKSPACE/fingerprint/responses/ | tee $WORKSPACE/fingerprint/rgf-all-$OUTPUT.txt", + "cmd": "$GO_PATH/rgf -dirir $WORKSPACE/fingerprint/responses/ | tee $WORKSPACE/fingerprint/rgf-all-$OUTPUT.txt", "output_path": "$WORKSPACE/fingerprint/rgf-all-$OUTPUT.txt", "std_path": "", }, @@ -273,13 +224,13 @@ class ScreenShot: 'general': [ { "banner": "aquatone", - "cmd": f"cat $WORKSPACE/probing/http-$OUTPUT.txt | $GO_PATH/aquatone -scan-timeout 1000 -threads {threads} -out $WORKSPACE/screenshot/$OUTPUT-aquatone", + "cmd": f"cat $WORKSPACE/fingerprint/success-http-$OUTPUT.txt | $GO_PATH/aquatone -threads {threads} -out $WORKSPACE/screenshot/$OUTPUT-aquatone", "output_path": "$WORKSPACE/screenshot/$OUTPUT-aquatone/aquatone_report.html", "std_path": "$WORKSPACE/screenshot/std-$OUTPUT-aquatone.std" }, { "banner": "gowitness", - "cmd": f"$GO_PATH/gowitness file -s $WORKSPACE/probing/http-$OUTPUT.txt -t {threads} --log-level fatal --destination $WORKSPACE/screenshot/raw-gowitness/ --db $WORKSPACE/screenshot/gowitness.db", + "cmd": f"$GO_PATH/gowitness file -s $WORKSPACE/fingerprint/success-http-$OUTPUT.txt -t {threads} --log-level fatal --destination $WORKSPACE/screenshot/raw-gowitness/ --db $WORKSPACE/screenshot/gowitness.db", "output_path": "$WORKSPACE/screenshot/gowitness.db", "std_path": "", }, @@ -309,7 +260,7 @@ class StoScan: }, { "path": "$WORKSPACE/stoscan/all-dig-info.txt", - "type": "bash" + "type": "final, bash" }, ] logs = [] @@ -317,7 +268,7 @@ class StoScan: 'general': [ { "banner": "tko-subs", - "cmd": "$GO_PATH/tko-subs -data $DATA_PATH/providers-data.csv -domains $WORKSPACE/probing/domains-$OUTPUT.txt -output $WORKSPACE/stoscan/takeover-$TARGET-tko-subs.txt", + "cmd": "$GO_PATH/tko-subs -data $DATA_PATH/providers-data.csv -domains $WORKSPACE/probing/resolved-$OUTPUT.txt -output $WORKSPACE/stoscan/takeover-$TARGET-tko-subs.txt", "output_path": "$WORKSPACE/stoscan/takeover-$TARGET-tko-subs.txt", "std_path": "$WORKSPACE/stoscan/std-takeover-$TARGET-tko-subs.std", }, @@ -326,10 +277,9 @@ class StoScan: "cmd": "$GO_PATH/subjack -v -m -c $DATA_PATH/fingerprints.json -w $WORKSPACE/probing/domains-$OUTPUT.txt -t 100 -timeout 30 -o $WORKSPACE/stoscan/takeover-$TARGET-subjack.txt -ssl", "output_path": "$WORKSPACE/stoscan/takeover-$TARGET-subjack.txt", "std_path": "$WORKSPACE/stoscan/std-takeover-$TARGET-subjack.std" - }, - { + }, { "banner": "massdns resolve IP", - "cmd": "cat $WORKSPACE/probing/resolved-$OUTPUT.txt | $PLUGINS_PATH/massdns/bin/massdns -r $PLUGINS_PATH/massdns/lists/resolvers.txt -q -t A -o F -w $WORKSPACE/stoscan/all-dig-info.txt", + "cmd": "cat $WORKSPACE/probing/raw-all-$OUTPUT.txt | $PLUGINS_PATH/massdns/bin/massdns -r $PLUGINS_PATH/massdns/lists/resolvers.txt -q -t A -o F -w $WORKSPACE/stoscan/all-dig-info.txt", "output_path": "$WORKSPACE/stoscan/all-dig-info.txt", "std_path": "", "waiting": "first", @@ -341,6 +291,13 @@ class StoScan: "output_path": "$WORKSPACE/stoscan/have-cname.txt", "std_path": "", }, + { + "requirement": "$WORKSPACE/stoscan/all-dig-info.txt", + "banner": "rgf extract Azure", + "cmd": "$GO_PATH/rgf -file $WORKSPACE/stoscan/all-dig-info.txt azure | tee $WORKSPACE/stoscan/azure-sto.txt", + "output_path": "$WORKSPACE/stoscan/azure-sto.txt", + "std_path": "", + }, ], } @@ -566,7 +523,7 @@ class DirbScan: }, { "banner": "ffuf dirscan", - "cmd": "$ALIAS_PATH/dirscan -i [[0]] -w '$DATA_PATH/wordlists/content/top10000.txt' -o '$WORKSPACE/directory/raw' -p '$GO_PATH' -s '$WORKSPACE/directory'", + "cmd": "$ALIAS_PATH/dirscan -i [[0]] -w '$DATA_PATH/wordlists/content/quick.txt' -o '$WORKSPACE/directory/raw' -p '$GO_PATH' -s '$WORKSPACE/directory'", "output_path": "", "std_path": "", "chunk": 5, @@ -576,7 +533,7 @@ class DirbScan: { "requirement": "$WORKSPACE/probing/http-$OUTPUT.txt", "banner": "csv beautify", - "cmd": "cat $WORKSPACE/directory/directory-summary.csv | csvlook --max-column-width 100 | tee $WORKSPACE/directory/beautify-summary.csv", + "cmd": "cat $WORKSPACE/directory/raw/* | csvcut -c 2-6 | csvlook | tee $WORKSPACE/directory/beautify-summary.csv", "output_path": "", "std_path": "", "waiting": "last",