WEBDIENSTE/rengine
1
0
Fork 0
mirror of https://github.com/yogeshojha/rengine.git synced 2026-09-22 01:34:55 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix celery, ready for release testing

Browse Source
This commit is contained in:
Yogesh Ojha
2023-09-15 16:58:41 +05:30
parent 36fb811ad4
commit 6a98a8717e
6 changed files with 50 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docker-compose.dev.yml
+1 -1
View File
@@ -27,7 +27,7 @@ services:
context: ./web
restart: always
entrypoint: /usr/src/app/celery-entrypoint.sh
# command: watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --autoscale=${MAX_CONCURRENCY},${MIN_CONCURRENCY} -l INFO
# command: watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --autoscale=10,0 -l INFO
volumes:
- ./web:/usr/src/app
- github_repos:/usr/src/github
web/celery-entrypoint.sh
+18 -34
View File
@@ -129,39 +129,23 @@ exec "$@"
echo 'alias httpx="/go/bin/httpx"' >> ~/.bashrc
# watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --autoscale=10,0 -l INFO -Q scan_queue &
echo "Starting Workers..."
watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=100 --pool=gevent --loglevel=info -Q initiate_scan_queue -n initiate_scan_worker &
watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=500 --pool=gevent --loglevel=info -Q subscan_queue -n subscan_worker &
watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=50 --pool=gevent --loglevel=info -Q report_queue -n report_worker &
watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=100 --pool=gevent --loglevel=info -Q subdomain_discovery_queue -n subdomain_discovery_worker &
watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=50 --pool=gevent --loglevel=info -Q osint_queue -n osint_worker &
watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=50 --pool=gevent --loglevel=info -Q osint_discovery_queue -n osint_discovery_worker &
watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=50 --pool=gevent --loglevel=info -Q dorking_queue -n dorking_worker &
watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=50 --pool=gevent --loglevel=info -Q theHarvester_queue -n theHarvester_worker &
watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=50 --pool=gevent --loglevel=info -Q h8mail_queue -n h8mail_worker &
watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=100 --pool=gevent --loglevel=info -Q screenshot_queue -n screenshot_worker &
watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=100 --pool=gevent --loglevel=info -Q port_scan_queue -n port_scan_worker &
watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=100 --pool=gevent --loglevel=info -Q nmap_queue -n nmap_worker &
watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=100 --pool=gevent --loglevel=info -Q waf_detection_queue -n waf_detection_worker &
watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=100 --pool=gevent --loglevel=info -Q dir_file_fuzz_queue -n dir_file_fuzz_worker &
watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=100 --pool=gevent --loglevel=info -Q fetch_url_queue -n fetch_url_worker &
watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=100 --pool=gevent --loglevel=info -Q vulnerability_scan_queue -n vulnerability_scan_worker &
watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=1000 --pool=gevent --loglevel=info -Q vulnerability_scan_module_queue -n vulnerability_scan_module_worker &
watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=100 --pool=gevent --loglevel=info -Q dalfox_xss_scan_queue -n dalfox_xss_scan_worker &
watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=100 --pool=gevent --loglevel=info -Q crlfuzz_queue -n crlfuzz_worker &
watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=5000 --pool=gevent --loglevel=info -Q http_crawl_queue -n http_crawl_worker &
watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=50 --pool=gevent --loglevel=info -Q send_notif_queue -n send_notif_worker &
watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=50 --pool=gevent --loglevel=info -Q send_scan_notif_queue -n send_scan_notif_worker &
watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=50 --pool=gevent --loglevel=info -Q send_task_notif_queue -n send_task_notif_worker &
watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=50 --pool=gevent --loglevel=info -Q send_file_to_discord_queue -n send_file_to_discord_worker &
watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=50 --pool=gevent --loglevel=info -Q send_hackerone_report_queue -n send_hackerone_report_worker &
watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=50 --pool=gevent --loglevel=info -Q parse_nmap_results_queue -n parse_nmap_results_worker &
watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=500 --pool=gevent --loglevel=info -Q geo_localize_queue -n geo_localize_worker &
watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=100 --pool=gevent --loglevel=info -Q query_whois_queue -n query_whois_worker &
watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=100 --pool=gevent --loglevel=info -Q remove_duplicate_endpoints_queue -n remove_duplicate_endpoints_worker &
watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=500 --pool=gevent --loglevel=info -Q run_command_queue -n run_command_worker &
watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=100 --pool=gevent --loglevel=info -Q query_reverse_whois_queue -n query_reverse_whois_worker &
watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=100 --pool=gevent --loglevel=info -Q query_ip_history_queue -n query_ip_history_worker &
watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=100 --pool=gevent --loglevel=info -Q gpt_queue -n gpt_worker &
watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=100 --pool=gevent --loglevel=info -Q s3scanner_queue -n s3scanner_worker
celery -A reNgine.tasks worker --autoscale=7,0 --loglevel=info -Q main_scan_queue &
celery -A reNgine.tasks worker --pool=gevent --concurrency=30 --loglevel=info -Q initiate_scan_queue -n initiate_scan_worker &
celery -A reNgine.tasks worker --pool=gevent --concurrency=30 --loglevel=info -Q subscan_queue -n subscan_worker &
celery -A reNgine.tasks worker --pool=gevent --concurrency=20 --loglevel=info -Q report_queue -n report_worker &
celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=info -Q send_notif_queue -n send_notif_worker &
celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=info -Q send_scan_notif_queue -n send_scan_notif_worker &
celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=info -Q send_task_notif_queue -n send_task_notif_worker &
celery -A reNgine.tasks worker --pool=gevent --concurrency=5 --loglevel=info -Q send_file_to_discord_queue -n send_file_to_discord_worker &
celery -A reNgine.tasks worker --pool=gevent --concurrency=5 --loglevel=info -Q send_hackerone_report_queue -n send_hackerone_report_worker &
celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=info -Q parse_nmap_results_queue -n parse_nmap_results_worker &
celery -A reNgine.tasks worker --pool=gevent --concurrency=20 --loglevel=info -Q geo_localize_queue -n geo_localize_worker &
celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=info -Q query_whois_queue -n query_whois_worker &
celery -A reNgine.tasks worker --pool=gevent --concurrency=30 --loglevel=info -Q remove_duplicate_endpoints_queue -n remove_duplicate_endpoints_worker &
celery -A reNgine.tasks worker --pool=gevent --concurrency=50 --loglevel=info -Q run_command_queue -n run_command_worker &
celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=info -Q query_reverse_whois_queue -n query_reverse_whois_worker &
celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=info -Q query_ip_history_queue -n query_ip_history_worker &
celery -A reNgine.tasks worker --pool=gevent --concurrency=30 --loglevel=info -Q gpt_queue -n gpt_worker
exec "$@"
web/reNgine/settings.py
+1 -1
View File
@@ -179,7 +179,7 @@ CELERY_TIMEZONE = 'UTC'
CELERY_IGNORE_RESULTS = False
CELERY_EAGER_PROPAGATES_EXCEPTIONS = True
CELERY_TRACK_STARTED = True
CELERY_BROKER_CONNECTION_RETRY_ON_STARTUP = True
'''
ROLES and PERMISSIONS
'''
web/reNgine/tasks.py
+19 -20
View File
@@ -365,7 +365,7 @@ def report(ctx={}, description=None):
# Tracked reNgine tasks #
#--------------------------#
@app.task(name='subdomain_discovery', queue='subdomain_discovery_queue', base=RengineTask, bind=True)
@app.task(name='subdomain_discovery', queue='main_scan_queue', base=RengineTask, bind=True)
def subdomain_discovery(
self,
host=None,
@@ -413,9 +413,8 @@ def subdomain_discovery(
# Run tools
for tool in tools:
cmd = None
logger.info(f'Scanning subdomains with {tool}')
logger.info(f'Scanning subdomains for {host} with {tool}')
proxy = get_random_proxy()
if tool in default_subdomain_tools:
if tool == 'amass-passive':
cmd = f'amass enum -passive -d {host} -o {self.results_dir}/subdomains_amass.txt'
@@ -579,7 +578,7 @@ def subdomain_discovery(
return SubdomainSerializer(subdomains, many=True).data
@app.task(name='osint', queue='osint_queue', base=RengineTask, bind=True)
@app.task(name='osint', queue='main_scan_queue', base=RengineTask, bind=True)
def osint(self, host=None, ctx={}, description=None):
"""Run Open-Source Intelligence tools on selected domain.
@@ -606,7 +605,7 @@ def osint(self, host=None, ctx={}, description=None):
return results
@app.task(name='osint_discovery', queue='osint_discovery_queue', base=RengineTask, bind=True)
@app.task(name='osint_discovery', queue='main_scan_queue', base=RengineTask, bind=True)
def osint_discovery(self, host=None, ctx={}):
"""Run OSInt discovery.
@@ -666,7 +665,7 @@ def osint_discovery(self, host=None, ctx={}):
return results
@app.task(name='dorking', queue='dorking_queue', base=RengineTask, bind=True)
@app.task(name='dorking', queue='main_scan_queue', base=RengineTask, bind=True)
def dorking(self, host=None, ctx={}):
"""Run Google dorks.
@@ -958,7 +957,7 @@ def dorking(self, host=None, ctx={}):
return results
@app.task(name='theHarvester', queue='theHarvester_queue', base=RengineTask, bind=True)
@app.task(name='theHarvester', queue='main_scan_queue', base=RengineTask, bind=True)
def theHarvester(self, host=None, ctx={}):
"""Run theHarvester to get save emails, hosts, employees found in domain.
@@ -1073,7 +1072,7 @@ def theHarvester(self, host=None, ctx={}):
return data
@app.task(name='h8mail', queue='h8mail_queue', base=RengineTask, bind=True)
@app.task(name='h8mail', queue='main_scan_queue', base=RengineTask, bind=True)
def h8mail(self, input_path=None, ctx={}):
"""Run h8mail.
@@ -1113,7 +1112,7 @@ def h8mail(self, input_path=None, ctx={}):
return creds
@app.task(name='screenshot', queue='screenshot_queue', base=RengineTask, bind=True)
@app.task(name='screenshot', queue='main_scan_queue', base=RengineTask, bind=True)
def screenshot(self, ctx={}, description=None):
"""Uses EyeWitness to gather screenshot of a domain and/or url.
@@ -1205,7 +1204,7 @@ def screenshot(self, ctx={}, description=None):
send_file_to_discord.delay(path, title)
@app.task(name='port_scan', queue='port_scan_queue', base=RengineTask, bind=True)
@app.task(name='port_scan', queue='main_scan_queue', base=RengineTask, bind=True)
def port_scan(self, hosts=[], ctx={}, description=None):
"""Run port scan.
@@ -1379,7 +1378,7 @@ def port_scan(self, hosts=[], ctx={}, description=None):
return ports_data
@app.task(name='nmap', queue='nmap_queue', base=RengineTask, bind=True)
@app.task(name='nmap', queue='main_scan_queue', base=RengineTask, bind=True)
def nmap(
self,
cmd=None,
@@ -1464,7 +1463,7 @@ def nmap(
return vulns
@app.task(name='waf_detection', queue='waf_detection_queue', base=RengineTask, bind=True)
@app.task(name='waf_detection', queue='main_scan_queue', base=RengineTask, bind=True)
def waf_detection(self, ctx={}, description=None):
"""
Uses wafw00f to check for the presence of a WAF.
@@ -1525,7 +1524,7 @@ def waf_detection(self, ctx={}, description=None):
return wafs
@app.task(name='dir_file_fuzz', queue='dir_file_fuzz_queue', base=RengineTask, bind=True)
@app.task(name='dir_file_fuzz', queue='main_scan_queue', base=RengineTask, bind=True)
def dir_file_fuzz(self, ctx={}, description=None):
"""Perform directory scan, and currently uses `ffuf` as a default tool.
@@ -1674,7 +1673,7 @@ def dir_file_fuzz(self, ctx={}, description=None):
return results
@app.task(name='fetch_url', queue='fetch_url_queue', base=RengineTask, bind=True)
@app.task(name='fetch_url', queue='main_scan_queue', base=RengineTask, bind=True)
def fetch_url(self, urls=[], ctx={}, description=None):
"""Fetch URLs using different tools like gauplus, gospider, waybackurls ...
@@ -1913,7 +1912,7 @@ def parse_curl_output(response):
}
@app.task(name='vulnerability_scan', queue='vulnerability_scan_queue', base=RengineTask, bind=True)
@app.task(name='vulnerability_scan', queue='main_scan_queue', base=RengineTask, bind=True)
def vulnerability_scan(self, urls=[], ctx={}, description=None):
"""HTTP vulnerability scan using `nuclei`.
@@ -2041,7 +2040,7 @@ def vulnerability_scan(self, urls=[], ctx={}, description=None):
# return results
return None
@app.task(name='vulnerability_scan_module', queue='vulnerability_scan_module_queue', base=RengineTask, bind=True)
@app.task(name='vulnerability_scan_module', queue='main_scan_queue', base=RengineTask, bind=True)
def vulnerability_scan_module(self, cmd, severity, enable_http_crawl, should_fetch_gpt_report, ctx={}, description=None):
'''
This celery task is supposed to run vulnerability scan in parallel.
@@ -2281,7 +2280,7 @@ def add_gpt_description_db(title, path, description, impact, remediation, refere
gpt_report.save()
@app.task(name='dalfox_xss_scan', queue='dalfox_xss_scan_queue', base=RengineTask, bind=True)
@app.task(name='dalfox_xss_scan', queue='main_scan_queue', base=RengineTask, bind=True)
def dalfox_xss_scan(self, urls=[], ctx={}, description=None):
"""XSS Scan using dalfox
@@ -2407,7 +2406,7 @@ def dalfox_xss_scan(self, urls=[], ctx={}, description=None):
return results
@app.task(name='crlfuzz', queue='crlfuzz_queue', base=RengineTask, bind=True)
@app.task(name='crlfuzz', queue='main_scan_queue', base=RengineTask, bind=True)
def crlfuzz(self, urls=[], ctx={}, description=None):
"""CRLF Fuzzing with CRLFuzz
@@ -2528,7 +2527,7 @@ def crlfuzz(self, urls=[], ctx={}, description=None):
return results
@app.task(name='s3scanner', queue='s3scanner_queue', base=RengineTask, bind=True)
@app.task(name='s3scanner', queue='main_scan_queue', base=RengineTask, bind=True)
def s3scanner(self, ctx={}, description=None):
"""Bucket Scanner
@@ -2560,7 +2559,7 @@ def s3scanner(self, ctx={}, description=None):
logger.info(f"s3 bucket added {result['provider']}-{result['name']}-{result['region']}")
@app.task(name='http_crawl', queue='http_crawl_queue', base=RengineTask, bind=True)
@app.task(name='http_crawl', queue='main_scan_queue', base=RengineTask, bind=True)
def http_crawl(
self,
urls=[],
web/requirements.txt
+8 -8
View File
@@ -1,7 +1,7 @@
aiodns==3.0.0
argh==0.26.2
beautifulsoup4==4.9.3
celery==5.2.7
celery==5.3.4
degoogle==1.0.1
discord-webhook==0.14.0
Django==3.2.4
@@ -16,28 +16,28 @@ djangorestframework-datatables==0.6.0
dotted-dict==1.1.3
drf-yasg==1.21.3
fuzzywuzzy==0.18.0
gunicorn==20.1.0
gevent==23.9.0.post1
gunicorn==21.2.0
gevent==23.9.1
humanize==4.3.0
Markdown==3.3.4
metafinder==1.1
netaddr==0.8.0
netlas==0.4.0
openai==0.27.4
openai==0.28.0
PyYAML==5.4.1
PySocks==1.7.1
psycopg2==2.9.1
psycopg2==2.9.7
pycvesearch==1.2
python-Levenshtein==0.12.2
PyVirtualDisplay==2.2
redis==3.5.3
requests==2.25.1
redis==5.0.0
requests==2.31.0
scapy==2.4.3
selenium==3.141.0
tldextract==3.1.0
uro==1.0.0
validators==0.18.2
watchdog==2.1.9
watchdog==3.0.0
whatportis==0.8.2
weasyprint==53.3
wafw00f==2.2.0
web/startScan/models.py
+3
View File
@@ -509,6 +509,9 @@ class Command(models.Model):
output = models.TextField(blank=True, null=True)
time = models.DateTimeField()
def __str__(self):
return str(self.command)
class Waf(models.Model):
id = models.AutoField(primary_key=True)