diff --git a/install.sh b/install.sh index 90c7d2b..10edf3b 100755 --- a/install.sh +++ b/install.sh @@ -24,6 +24,7 @@ install_banner "git, nmap, masscan, chromium, npm, golang" [ -x "$(command -v csvlook)" ] || sudo $PACKGE_MANAGER install csvkit -y 2>/dev/null [ -x "$(command -v ripgrep)" ] || sudo $PACKGE_MANAGER install ripgrep -y 2>/dev/null [ -x "$(command -v unzip)" ] || sudo $PACKGE_MANAGER install unzip -y 2>/dev/null +[ -x "$(command -v chromium-browser)" ] || sudo $PACKGE_MANAGER install chromium-browser -y 2>/dev/null #### Download stuff directly install_banner "wordlists" @@ -207,6 +208,9 @@ install_banner "Metabigor" git clone https://github.com/j3ssie/Metabigor 2>/dev/null pip3 install -r Metabigor/requirements.txt +install_banner "rgf" +git clone https://github.com/j3ssie/rgf 2>/dev/null + install_banner "dirsearch" git clone https://github.com/maurosoria/dirsearch 2>/dev/null diff --git a/lib/workflow/direct.py b/lib/workflow/direct.py index 505879c..01e0ba3 100644 --- a/lib/workflow/direct.py +++ b/lib/workflow/direct.py @@ -189,7 +189,7 @@ class LinkFinding: "note": "final", }, { - "path": "$WORKSPACE/links/waybackurls-$OUTPUT.txt", + "path": "$WORKSPACE/links/raw-wayback-$OUTPUT.txt", "type": "bash", "note": "final", } @@ -207,18 +207,26 @@ class LinkFinding: "cleaned_output": "$WORKSPACE/links/waybackurls-$OUTPUT.txt", }, { - "requirement": "$WORKSPACE/probing/domain-$OUTPUT.txt", + "requirement": "$WORKSPACE/probing/http-$OUTPUT.txt", "pre_run": "get_domains", "banner": "linkfinder", - "cmd": "python3 $PLUGINS_PATH/LinkFinder/linkfinder.py -i [[0]] -d -o cli | tee $WORKSPACE/links/raw/[[1]]-$OUTPUT.txt", - "output_path": "tee $WORKSPACE/links/raw/[[1]]-$OUTPUT.txt", + "cmd": "$ALIAS_PATH/linkfinding -i '[[0]]' -o '$WORKSPACE/links/raw/' -s '$WORKSPACE/links/summary-$OUTPUT.txt' -p '$PLUGINS_PATH'", + "output_path": "$WORKSPACE/links/raw/[[0]]-$OUTPUT.txt", "std_path": "", "chunk": 5, "cmd_type": "list", - "resources": "l0|$WORKSPACE/probing/http-$OUTPUT.txt;;l1|$WORKSPACE/probing/domain-$OUTPUT.txt", + "resources": "l0|$WORKSPACE/probing/http-$OUTPUT.txt", "post_run": "clean_linkfinder", "cleaned_output": "$WORKSPACE/links/summary-$OUTPUT.txt", }, + { + "requirement": "$WORKSPACE/links/raw-wayback-$OUTPUT.txt", + "banner": "Formatting Input", + "cmd": "cat $WORKSPACE/links/raw-wayback-$OUTPUT.txt | unfurl -u format %d%p", + "output_path": "$WORKSPACE/links/$OUTPUT-paths.txt", + "std_path": "", + "waiting": "last", + }, ], } diff --git a/lib/workflow/general.py b/lib/workflow/general.py index 10a0035..e766681 100644 --- a/lib/workflow/general.py +++ b/lib/workflow/general.py @@ -1,6 +1,10 @@ ''' Storing all pre-defined commands ''' +import multiprocessing + +cpu_cores = multiprocessing.cpu_count() +threads = str(cpu_cores * 2) class Sample: @@ -229,7 +233,7 @@ class Fingerprint: 'general': [ { "banner": "webanalyze", - "cmd": "$GO_PATH/webanalyze -apps $DATA_PATH/apps.json -hosts $WORKSPACE/probing/http-$OUTPUT.txt -output json -worker 20 | tee $WORKSPACE/fingerprint/$OUTPUT-technology.json", + "cmd": f"$GO_PATH/webanalyze -apps $DATA_PATH/apps.json -hosts $WORKSPACE/probing/http-$OUTPUT.txt -output json -worker {threads} | tee $WORKSPACE/fingerprint/$OUTPUT-technology.json", "output_path": "$WORKSPACE/fingerprint/$OUTPUT-technology.json", "std_path": "$WORKSPACE/fingerprint/std-$OUTPUT-technology.std", "post_run": "update_tech", @@ -241,6 +245,13 @@ class Fingerprint: "output_path": "$WORKSPACE/fingerprint/responses/index", "std_path": "", }, + { + "requirement": "$WORKSPACE/fingerprint/responses/index", + "banner": "rgf extract all", + "cmd": "$PLUGINS_PATH/rgf/rgf.py -d $WORKSPACE/fingerprint/responses/ | tee $WORKSPACE/fingerprint/rgf-all-$OUTPUT.txt", + "output_path": "$WORKSPACE/fingerprint/rgf-all-$OUTPUT.txt", + "std_path": "", + }, ], } @@ -263,13 +274,13 @@ class ScreenShot: 'general': [ { "banner": "aquatone", - "cmd": "cat $WORKSPACE/probing/resolved-$OUTPUT.txt | $GO_PATH/aquatone -scan-timeout 1000 -threads 20 -out $WORKSPACE/screenshot/$OUTPUT-aquatone", + "cmd": f"cat $WORKSPACE/probing/resolved-$OUTPUT.txt | $GO_PATH/aquatone -scan-timeout 1000 -threads {threads} -out $WORKSPACE/screenshot/$OUTPUT-aquatone", "output_path": "$WORKSPACE/screenshot/$OUTPUT-aquatone/aquatone_report.html", "std_path": "$WORKSPACE/screenshot/std-$OUTPUT-aquatone.std" }, { "banner": "gowitness", - "cmd": "$GO_PATH/gowitness file -s $WORKSPACE/probing/http-$OUTPUT.txt -t 30 --log-level fatal --destination $WORKSPACE/screenshot/raw-gowitness/ --db $WORKSPACE/screenshot/gowitness.db", + "cmd": f"$GO_PATH/gowitness file -s $WORKSPACE/probing/http-$OUTPUT.txt -t {threads} --log-level fatal --destination $WORKSPACE/screenshot/raw-gowitness/ --db $WORKSPACE/screenshot/gowitness.db", "output_path": "$WORKSPACE/screenshot/gowitness.db", "std_path": "", }, @@ -322,9 +333,14 @@ class StoScan: "cmd": "cat $WORKSPACE/probing/resolved-$OUTPUT.txt | $PLUGINS_PATH/massdns/bin/massdns -r $PLUGINS_PATH/massdns/lists/resolvers.txt -q -t A -o F -w $WORKSPACE/stoscan/all-dig-info.txt", "output_path": "$WORKSPACE/stoscan/all-dig-info.txt", "std_path": "", - "waiting": "last", - # "post_run": "grepping_sign", - # "cleaned_output": "$WORKSPACE/probing/really-final-$OUTPUT.txt", + "waiting": "first", + }, + { + "requirement": "$WORKSPACE/stoscan/all-dig-info.txt", + "banner": "rgf extract CNAME", + "cmd": "$PLUGINS_PATH/rgf/rgf.py -s cname -f $WORKSPACE/stoscan/all-dig-info.txt -o $WORKSPACE/stoscan/have-cname.txt", + "output_path": "$WORKSPACE/stoscan/have-cname.txt", + "std_path": "", }, ], } diff --git a/osmedeus.py b/osmedeus.py index da8bd08..831a6d3 100755 --- a/osmedeus.py +++ b/osmedeus.py @@ -126,7 +126,7 @@ def main(): parser.add_argument('--reset', action='store_true', help='Clean configurations') parser.add_argument('-hh', '--helps', dest='helps', - action='store_true', help='Display more help messaage') + action='store_true', help='Display more help message') args = parser.parse_args() if len(sys.argv) == 1: