mirror of
https://github.com/yogeshojha/rengine.git
synced 2026-09-27 04:04:52 +02:00
267 lines
8.4 KiB
Bash
Executable File
267 lines
8.4 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# apply existing migrations
|
|
python3 manage.py migrate
|
|
|
|
# make migrations for specific apps
|
|
apps=(
|
|
"targetApp"
|
|
"scanEngine"
|
|
"startScan"
|
|
"dashboard"
|
|
"recon_note"
|
|
)
|
|
|
|
create_migrations() {
|
|
local app=$1
|
|
echo "Creating migrations for $app..."
|
|
python3 manage.py makemigrations $app
|
|
echo "Finished creating migrations for $app"
|
|
echo "----------------------------------------"
|
|
}
|
|
|
|
echo "Starting migration creation process..."
|
|
|
|
for app in "${apps[@]}"
|
|
do
|
|
create_migrations $app
|
|
done
|
|
|
|
echo "Migration creation process completed."
|
|
|
|
# apply migrations again
|
|
echo "Applying migrations..."
|
|
python3 manage.py migrate
|
|
echo "Migration process completed."
|
|
|
|
|
|
python3 manage.py collectstatic --no-input --clear
|
|
|
|
# Load default engines, keywords, and external tools
|
|
python3 manage.py loaddata fixtures/default_scan_engines.yaml --app scanEngine.EngineType
|
|
python3 manage.py loaddata fixtures/default_keywords.yaml --app scanEngine.InterestingLookupModel
|
|
python3 manage.py loaddata fixtures/external_tools.yaml --app scanEngine.InstalledExternalTool
|
|
|
|
# install firefox https://askubuntu.com/a/1404401
|
|
echo '
|
|
Package: *
|
|
Pin: release o=LP-PPA-mozillateam
|
|
Pin-Priority: 1001
|
|
|
|
Package: firefox
|
|
Pin: version 1:1snap1-0ubuntu2
|
|
Pin-Priority: -1
|
|
' | tee /etc/apt/preferences.d/mozilla-firefox
|
|
apt update
|
|
apt install firefox -y
|
|
|
|
# Temporary fix for whatportis bug - See https://github.com/yogeshojha/rengine/issues/984
|
|
sed -i 's/purge()/truncate()/g' /usr/local/lib/python3.10/dist-packages/whatportis/cli.py
|
|
|
|
# update whatportis
|
|
yes | whatportis --update
|
|
|
|
# clone dirsearch default wordlist
|
|
if [ ! -d "/usr/src/wordlist" ]
|
|
then
|
|
echo "Making Wordlist directory"
|
|
mkdir /usr/src/wordlist
|
|
fi
|
|
|
|
if [ ! -f "/usr/src/wordlist/" ]
|
|
then
|
|
echo "Downloading Default Directory Bruteforce Wordlist"
|
|
wget https://raw.githubusercontent.com/maurosoria/dirsearch/master/db/dicc.txt -O /usr/src/wordlist/dicc.txt
|
|
fi
|
|
|
|
# check if default wordlist for amass exists
|
|
if [ ! -f /usr/src/wordlist/deepmagic.com-prefixes-top50000.txt ];
|
|
then
|
|
echo "Downloading Deepmagic top 50000 Wordlist"
|
|
wget https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/DNS/deepmagic.com-prefixes-top50000.txt -O /usr/src/wordlist/deepmagic.com-prefixes-top50000.txt
|
|
fi
|
|
|
|
# clone Sublist3r
|
|
if [ ! -d "/usr/src/github/Sublist3r" ]
|
|
then
|
|
echo "Cloning Sublist3r"
|
|
git clone https://github.com/aboul3la/Sublist3r /usr/src/github/Sublist3r
|
|
fi
|
|
python3 -m pip install -r /usr/src/github/Sublist3r/requirements.txt
|
|
|
|
# clone OneForAll
|
|
if [ ! -d "/usr/src/github/OneForAll" ]
|
|
then
|
|
echo "Cloning OneForAll"
|
|
git clone https://github.com/shmilylty/OneForAll /usr/src/github/OneForAll
|
|
fi
|
|
python3 -m pip install -r /usr/src/github/OneForAll/requirements.txt
|
|
|
|
# clone eyewitness
|
|
if [ ! -d "/usr/src/github/EyeWitness" ]
|
|
then
|
|
echo "Cloning EyeWitness"
|
|
git clone https://github.com/FortyNorthSecurity/EyeWitness /usr/src/github/EyeWitness
|
|
# pip install -r /usr/src/github/Eyewitness/requirements.txt
|
|
fi
|
|
|
|
# clone theHarvester
|
|
if [ ! -d "/usr/src/github/theHarvester" ]
|
|
then
|
|
echo "Cloning theHarvester"
|
|
git clone https://github.com/laramies/theHarvester /usr/src/github/theHarvester
|
|
fi
|
|
python3 -m pip install -r /usr/src/github/theHarvester/requirements/base.txt
|
|
|
|
# clone vulscan
|
|
if [ ! -d "/usr/src/github/scipag_vulscan" ]
|
|
then
|
|
echo "Cloning Nmap Vulscan script"
|
|
git clone https://github.com/scipag/vulscan /usr/src/github/scipag_vulscan
|
|
echo "Symlinking to nmap script dir"
|
|
ln -s /usr/src/github/scipag_vulscan /usr/share/nmap/scripts/vulscan
|
|
echo "Usage in reNgine, set vulscan/vulscan.nse in nmap_script scanEngine port_scan config parameter"
|
|
fi
|
|
|
|
# install h8mail
|
|
python3 -m pip install h8mail
|
|
|
|
# install gf patterns
|
|
if [ ! -d "/root/Gf-Patterns" ];
|
|
then
|
|
echo "Installing GF Patterns"
|
|
mkdir ~/.gf
|
|
cp -r $GOPATH/src/github.com/tomnomnom/gf/examples/*.json ~/.gf
|
|
git clone https://github.com/1ndianl33t/Gf-Patterns ~/Gf-Patterns
|
|
mv ~/Gf-Patterns/*.json ~/.gf
|
|
fi
|
|
|
|
# store scan_results
|
|
if [ ! -d "/usr/src/scan_results" ]
|
|
then
|
|
mkdir /usr/src/scan_results
|
|
fi
|
|
|
|
# test tools, required for configuration
|
|
naabu && subfinder && amass
|
|
nuclei
|
|
|
|
if [ ! -d "/root/nuclei-templates/geeknik_nuclei_templates" ];
|
|
then
|
|
echo "Installing Geeknik Nuclei templates"
|
|
git clone https://github.com/geeknik/the-nuclei-templates.git ~/nuclei-templates/geeknik_nuclei_templates
|
|
else
|
|
echo "Removing old Geeknik Nuclei templates and updating new one"
|
|
rm -rf ~/nuclei-templates/geeknik_nuclei_templates
|
|
git clone https://github.com/geeknik/the-nuclei-templates.git ~/nuclei-templates/geeknik_nuclei_templates
|
|
fi
|
|
|
|
if [ ! -f ~/nuclei-templates/ssrf_nagli.yaml ];
|
|
then
|
|
echo "Downloading ssrf_nagli for Nuclei"
|
|
wget https://raw.githubusercontent.com/NagliNagli/BountyTricks/main/ssrf.yaml -O ~/nuclei-templates/ssrf_nagli.yaml
|
|
fi
|
|
|
|
if [ ! -d "/usr/src/github/CMSeeK" ]
|
|
then
|
|
echo "Cloning CMSeeK"
|
|
git clone https://github.com/Tuhinshubhra/CMSeeK /usr/src/github/CMSeeK
|
|
pip install -r /usr/src/github/CMSeeK/requirements.txt
|
|
fi
|
|
|
|
# clone ctfr
|
|
if [ ! -d "/usr/src/github/ctfr" ]
|
|
then
|
|
echo "Cloning CTFR"
|
|
git clone https://github.com/UnaPibaGeek/ctfr /usr/src/github/ctfr
|
|
fi
|
|
|
|
# clone gooFuzz
|
|
if [ ! -d "/usr/src/github/goofuzz" ]
|
|
then
|
|
echo "Cloning GooFuzz"
|
|
git clone https://github.com/m3n0sd0n4ld/GooFuzz.git /usr/src/github/goofuzz
|
|
chmod +x /usr/src/github/goofuzz/GooFuzz
|
|
fi
|
|
|
|
# httpx seems to have issue, use alias instead!!!
|
|
echo 'alias httpx="/go/bin/httpx"' >> ~/.bashrc
|
|
|
|
# TEMPORARY FIX, httpcore is causing issues with celery, removing it as temp fix
|
|
#python3 -m pip uninstall -y httpcore
|
|
|
|
# TEMPORARY FIX FOR langchain
|
|
pip install tenacity==8.2.2
|
|
|
|
loglevel='info'
|
|
if [ "$DEBUG" == "1" ]; then
|
|
loglevel='debug'
|
|
fi
|
|
|
|
generate_worker_command() {
|
|
local queue=$1
|
|
local concurrency=$2
|
|
local worker_name=$3
|
|
local app=${4:-"reNgine.tasks"}
|
|
local directory=${5:-"/usr/src/app/reNgine/"}
|
|
|
|
local base_command="celery -A $app worker --pool=gevent --optimization=fair --autoscale=$concurrency,1 --loglevel=$loglevel -Q $queue -n $worker_name"
|
|
|
|
if [ "$DEBUG" == "1" ]; then
|
|
echo "watchmedo auto-restart --recursive --pattern=\"*.py\" --directory=\"$directory\" -- $base_command &"
|
|
else
|
|
echo "$base_command &"
|
|
fi
|
|
}
|
|
|
|
echo "Starting Celery Workers..."
|
|
|
|
commands=""
|
|
|
|
# Main scan worker
|
|
if [ "$DEBUG" == "1" ]; then
|
|
commands+="watchmedo auto-restart --recursive --pattern=\"*.py\" --directory=\"/usr/src/app/reNgine/\" -- celery -A reNgine.tasks worker --loglevel=$loglevel --optimization=fair --autoscale=$MAX_CONCURRENCY,$MIN_CONCURRENCY -Q main_scan_queue &"$'\n'
|
|
else
|
|
commands+="celery -A reNgine.tasks worker --loglevel=$loglevel --optimization=fair --autoscale=$MAX_CONCURRENCY,$MIN_CONCURRENCY -Q main_scan_queue &"$'\n'
|
|
fi
|
|
|
|
# API shared task worker
|
|
if [ "$DEBUG" == "1" ]; then
|
|
commands+="watchmedo auto-restart --recursive --pattern=\"*.py\" --directory=\"/usr/src/app/api/\" -- celery -A api.shared_api_tasks worker --pool=gevent --optimization=fair --concurrency=30 --loglevel=$loglevel -Q api_queue -n api_worker &"$'\n'
|
|
else
|
|
commands+="celery -A api.shared_api_tasks worker --pool=gevent --concurrency=30 --optimization=fair --loglevel=$loglevel -Q api_queue -n api_worker &"$'\n'
|
|
fi
|
|
|
|
# worker format: "queue_name:concurrency:worker_name"
|
|
workers=(
|
|
"initiate_scan_queue:30:initiate_scan_worker"
|
|
"subscan_queue:30:subscan_worker"
|
|
"report_queue:20:report_worker"
|
|
"send_notif_queue:10:send_notif_worker"
|
|
"send_task_notif_queue:10:send_task_notif_worker"
|
|
"send_file_to_discord_queue:5:send_file_to_discord_worker"
|
|
"send_hackerone_report_queue:5:send_hackerone_report_worker"
|
|
"parse_nmap_results_queue:10:parse_nmap_results_worker"
|
|
"geo_localize_queue:20:geo_localize_worker"
|
|
"query_whois_queue:10:query_whois_worker"
|
|
"remove_duplicate_endpoints_queue:30:remove_duplicate_endpoints_worker"
|
|
"run_command_queue:50:run_command_worker"
|
|
"query_reverse_whois_queue:10:query_reverse_whois_worker"
|
|
"query_ip_history_queue:10:query_ip_history_worker"
|
|
"llm_queue:30:llm_worker"
|
|
"dorking_queue:10:dorking_worker"
|
|
"osint_discovery_queue:10:osint_discovery_worker"
|
|
"h8mail_queue:10:h8mail_worker"
|
|
"theHarvester_queue:10:theHarvester_worker"
|
|
"send_scan_notif_queue:10:send_scan_notif_worker"
|
|
)
|
|
|
|
for worker in "${workers[@]}"; do
|
|
IFS=':' read -r queue concurrency worker_name <<< "$worker"
|
|
commands+="$(generate_worker_command "$queue" "$concurrency" "$worker_name")"$'\n'
|
|
done
|
|
commands="${commands%&}"
|
|
|
|
eval "$commands"
|
|
|
|
wait |