From 6a98a8717eb975d5784f1630dbdcfdf82bdf1e4e Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Fri, 15 Sep 2023 16:58:41 +0530 Subject: [PATCH] fix celery, ready for release testing --- docker-compose.dev.yml | 2 +- web/celery-entrypoint.sh | 52 ++++++++++++++-------------------------- web/reNgine/settings.py | 2 +- web/reNgine/tasks.py | 39 +++++++++++++++--------------- web/requirements.txt | 16 ++++++------- web/startScan/models.py | 3 +++ 6 files changed, 50 insertions(+), 64 deletions(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index b338a1dd..2a537d7f 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -27,7 +27,7 @@ services: context: ./web restart: always entrypoint: /usr/src/app/celery-entrypoint.sh - # command: watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --autoscale=${MAX_CONCURRENCY},${MIN_CONCURRENCY} -l INFO + # command: watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --autoscale=10,0 -l INFO volumes: - ./web:/usr/src/app - github_repos:/usr/src/github diff --git a/web/celery-entrypoint.sh b/web/celery-entrypoint.sh index 616e0175..cf3b0b44 100755 --- a/web/celery-entrypoint.sh +++ b/web/celery-entrypoint.sh @@ -129,39 +129,23 @@ exec "$@" echo 'alias httpx="/go/bin/httpx"' >> ~/.bashrc +# watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --autoscale=10,0 -l INFO -Q scan_queue & echo "Starting Workers..." -watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=100 --pool=gevent --loglevel=info -Q initiate_scan_queue -n initiate_scan_worker & -watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=500 --pool=gevent --loglevel=info -Q subscan_queue -n subscan_worker & -watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=50 --pool=gevent --loglevel=info -Q report_queue -n report_worker & -watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=100 --pool=gevent --loglevel=info -Q subdomain_discovery_queue -n subdomain_discovery_worker & -watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=50 --pool=gevent --loglevel=info -Q osint_queue -n osint_worker & -watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=50 --pool=gevent --loglevel=info -Q osint_discovery_queue -n osint_discovery_worker & -watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=50 --pool=gevent --loglevel=info -Q dorking_queue -n dorking_worker & -watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=50 --pool=gevent --loglevel=info -Q theHarvester_queue -n theHarvester_worker & -watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=50 --pool=gevent --loglevel=info -Q h8mail_queue -n h8mail_worker & -watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=100 --pool=gevent --loglevel=info -Q screenshot_queue -n screenshot_worker & -watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=100 --pool=gevent --loglevel=info -Q port_scan_queue -n port_scan_worker & -watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=100 --pool=gevent --loglevel=info -Q nmap_queue -n nmap_worker & -watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=100 --pool=gevent --loglevel=info -Q waf_detection_queue -n waf_detection_worker & -watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=100 --pool=gevent --loglevel=info -Q dir_file_fuzz_queue -n dir_file_fuzz_worker & -watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=100 --pool=gevent --loglevel=info -Q fetch_url_queue -n fetch_url_worker & -watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=100 --pool=gevent --loglevel=info -Q vulnerability_scan_queue -n vulnerability_scan_worker & -watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=1000 --pool=gevent --loglevel=info -Q vulnerability_scan_module_queue -n vulnerability_scan_module_worker & -watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=100 --pool=gevent --loglevel=info -Q dalfox_xss_scan_queue -n dalfox_xss_scan_worker & -watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=100 --pool=gevent --loglevel=info -Q crlfuzz_queue -n crlfuzz_worker & -watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=5000 --pool=gevent --loglevel=info -Q http_crawl_queue -n http_crawl_worker & -watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=50 --pool=gevent --loglevel=info -Q send_notif_queue -n send_notif_worker & -watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=50 --pool=gevent --loglevel=info -Q send_scan_notif_queue -n send_scan_notif_worker & -watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=50 --pool=gevent --loglevel=info -Q send_task_notif_queue -n send_task_notif_worker & -watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=50 --pool=gevent --loglevel=info -Q send_file_to_discord_queue -n send_file_to_discord_worker & -watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=50 --pool=gevent --loglevel=info -Q send_hackerone_report_queue -n send_hackerone_report_worker & -watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=50 --pool=gevent --loglevel=info -Q parse_nmap_results_queue -n parse_nmap_results_worker & -watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=500 --pool=gevent --loglevel=info -Q geo_localize_queue -n geo_localize_worker & -watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=100 --pool=gevent --loglevel=info -Q query_whois_queue -n query_whois_worker & -watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=100 --pool=gevent --loglevel=info -Q remove_duplicate_endpoints_queue -n remove_duplicate_endpoints_worker & -watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=500 --pool=gevent --loglevel=info -Q run_command_queue -n run_command_worker & -watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=100 --pool=gevent --loglevel=info -Q query_reverse_whois_queue -n query_reverse_whois_worker & -watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=100 --pool=gevent --loglevel=info -Q query_ip_history_queue -n query_ip_history_worker & -watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=100 --pool=gevent --loglevel=info -Q gpt_queue -n gpt_worker & -watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine worker --concurrency=100 --pool=gevent --loglevel=info -Q s3scanner_queue -n s3scanner_worker +celery -A reNgine.tasks worker --autoscale=7,0 --loglevel=info -Q main_scan_queue & +celery -A reNgine.tasks worker --pool=gevent --concurrency=30 --loglevel=info -Q initiate_scan_queue -n initiate_scan_worker & +celery -A reNgine.tasks worker --pool=gevent --concurrency=30 --loglevel=info -Q subscan_queue -n subscan_worker & +celery -A reNgine.tasks worker --pool=gevent --concurrency=20 --loglevel=info -Q report_queue -n report_worker & +celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=info -Q send_notif_queue -n send_notif_worker & +celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=info -Q send_scan_notif_queue -n send_scan_notif_worker & +celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=info -Q send_task_notif_queue -n send_task_notif_worker & +celery -A reNgine.tasks worker --pool=gevent --concurrency=5 --loglevel=info -Q send_file_to_discord_queue -n send_file_to_discord_worker & +celery -A reNgine.tasks worker --pool=gevent --concurrency=5 --loglevel=info -Q send_hackerone_report_queue -n send_hackerone_report_worker & +celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=info -Q parse_nmap_results_queue -n parse_nmap_results_worker & +celery -A reNgine.tasks worker --pool=gevent --concurrency=20 --loglevel=info -Q geo_localize_queue -n geo_localize_worker & +celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=info -Q query_whois_queue -n query_whois_worker & +celery -A reNgine.tasks worker --pool=gevent --concurrency=30 --loglevel=info -Q remove_duplicate_endpoints_queue -n remove_duplicate_endpoints_worker & +celery -A reNgine.tasks worker --pool=gevent --concurrency=50 --loglevel=info -Q run_command_queue -n run_command_worker & +celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=info -Q query_reverse_whois_queue -n query_reverse_whois_worker & +celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=info -Q query_ip_history_queue -n query_ip_history_worker & +celery -A reNgine.tasks worker --pool=gevent --concurrency=30 --loglevel=info -Q gpt_queue -n gpt_worker exec "$@" diff --git a/web/reNgine/settings.py b/web/reNgine/settings.py index 233726be..4fb209ac 100644 --- a/web/reNgine/settings.py +++ b/web/reNgine/settings.py @@ -179,7 +179,7 @@ CELERY_TIMEZONE = 'UTC' CELERY_IGNORE_RESULTS = False CELERY_EAGER_PROPAGATES_EXCEPTIONS = True CELERY_TRACK_STARTED = True - +CELERY_BROKER_CONNECTION_RETRY_ON_STARTUP = True ''' ROLES and PERMISSIONS ''' diff --git a/web/reNgine/tasks.py b/web/reNgine/tasks.py index de4ddfa3..a0bf7f4b 100644 --- a/web/reNgine/tasks.py +++ b/web/reNgine/tasks.py @@ -365,7 +365,7 @@ def report(ctx={}, description=None): # Tracked reNgine tasks # #--------------------------# -@app.task(name='subdomain_discovery', queue='subdomain_discovery_queue', base=RengineTask, bind=True) +@app.task(name='subdomain_discovery', queue='main_scan_queue', base=RengineTask, bind=True) def subdomain_discovery( self, host=None, @@ -413,9 +413,8 @@ def subdomain_discovery( # Run tools for tool in tools: cmd = None - logger.info(f'Scanning subdomains with {tool}') + logger.info(f'Scanning subdomains for {host} with {tool}') proxy = get_random_proxy() - if tool in default_subdomain_tools: if tool == 'amass-passive': cmd = f'amass enum -passive -d {host} -o {self.results_dir}/subdomains_amass.txt' @@ -579,7 +578,7 @@ def subdomain_discovery( return SubdomainSerializer(subdomains, many=True).data -@app.task(name='osint', queue='osint_queue', base=RengineTask, bind=True) +@app.task(name='osint', queue='main_scan_queue', base=RengineTask, bind=True) def osint(self, host=None, ctx={}, description=None): """Run Open-Source Intelligence tools on selected domain. @@ -606,7 +605,7 @@ def osint(self, host=None, ctx={}, description=None): return results -@app.task(name='osint_discovery', queue='osint_discovery_queue', base=RengineTask, bind=True) +@app.task(name='osint_discovery', queue='main_scan_queue', base=RengineTask, bind=True) def osint_discovery(self, host=None, ctx={}): """Run OSInt discovery. @@ -666,7 +665,7 @@ def osint_discovery(self, host=None, ctx={}): return results -@app.task(name='dorking', queue='dorking_queue', base=RengineTask, bind=True) +@app.task(name='dorking', queue='main_scan_queue', base=RengineTask, bind=True) def dorking(self, host=None, ctx={}): """Run Google dorks. @@ -958,7 +957,7 @@ def dorking(self, host=None, ctx={}): return results -@app.task(name='theHarvester', queue='theHarvester_queue', base=RengineTask, bind=True) +@app.task(name='theHarvester', queue='main_scan_queue', base=RengineTask, bind=True) def theHarvester(self, host=None, ctx={}): """Run theHarvester to get save emails, hosts, employees found in domain. @@ -1073,7 +1072,7 @@ def theHarvester(self, host=None, ctx={}): return data -@app.task(name='h8mail', queue='h8mail_queue', base=RengineTask, bind=True) +@app.task(name='h8mail', queue='main_scan_queue', base=RengineTask, bind=True) def h8mail(self, input_path=None, ctx={}): """Run h8mail. @@ -1113,7 +1112,7 @@ def h8mail(self, input_path=None, ctx={}): return creds -@app.task(name='screenshot', queue='screenshot_queue', base=RengineTask, bind=True) +@app.task(name='screenshot', queue='main_scan_queue', base=RengineTask, bind=True) def screenshot(self, ctx={}, description=None): """Uses EyeWitness to gather screenshot of a domain and/or url. @@ -1205,7 +1204,7 @@ def screenshot(self, ctx={}, description=None): send_file_to_discord.delay(path, title) -@app.task(name='port_scan', queue='port_scan_queue', base=RengineTask, bind=True) +@app.task(name='port_scan', queue='main_scan_queue', base=RengineTask, bind=True) def port_scan(self, hosts=[], ctx={}, description=None): """Run port scan. @@ -1379,7 +1378,7 @@ def port_scan(self, hosts=[], ctx={}, description=None): return ports_data -@app.task(name='nmap', queue='nmap_queue', base=RengineTask, bind=True) +@app.task(name='nmap', queue='main_scan_queue', base=RengineTask, bind=True) def nmap( self, cmd=None, @@ -1464,7 +1463,7 @@ def nmap( return vulns -@app.task(name='waf_detection', queue='waf_detection_queue', base=RengineTask, bind=True) +@app.task(name='waf_detection', queue='main_scan_queue', base=RengineTask, bind=True) def waf_detection(self, ctx={}, description=None): """ Uses wafw00f to check for the presence of a WAF. @@ -1525,7 +1524,7 @@ def waf_detection(self, ctx={}, description=None): return wafs -@app.task(name='dir_file_fuzz', queue='dir_file_fuzz_queue', base=RengineTask, bind=True) +@app.task(name='dir_file_fuzz', queue='main_scan_queue', base=RengineTask, bind=True) def dir_file_fuzz(self, ctx={}, description=None): """Perform directory scan, and currently uses `ffuf` as a default tool. @@ -1674,7 +1673,7 @@ def dir_file_fuzz(self, ctx={}, description=None): return results -@app.task(name='fetch_url', queue='fetch_url_queue', base=RengineTask, bind=True) +@app.task(name='fetch_url', queue='main_scan_queue', base=RengineTask, bind=True) def fetch_url(self, urls=[], ctx={}, description=None): """Fetch URLs using different tools like gauplus, gospider, waybackurls ... @@ -1913,7 +1912,7 @@ def parse_curl_output(response): } -@app.task(name='vulnerability_scan', queue='vulnerability_scan_queue', base=RengineTask, bind=True) +@app.task(name='vulnerability_scan', queue='main_scan_queue', base=RengineTask, bind=True) def vulnerability_scan(self, urls=[], ctx={}, description=None): """HTTP vulnerability scan using `nuclei`. @@ -2041,7 +2040,7 @@ def vulnerability_scan(self, urls=[], ctx={}, description=None): # return results return None -@app.task(name='vulnerability_scan_module', queue='vulnerability_scan_module_queue', base=RengineTask, bind=True) +@app.task(name='vulnerability_scan_module', queue='main_scan_queue', base=RengineTask, bind=True) def vulnerability_scan_module(self, cmd, severity, enable_http_crawl, should_fetch_gpt_report, ctx={}, description=None): ''' This celery task is supposed to run vulnerability scan in parallel. @@ -2281,7 +2280,7 @@ def add_gpt_description_db(title, path, description, impact, remediation, refere gpt_report.save() -@app.task(name='dalfox_xss_scan', queue='dalfox_xss_scan_queue', base=RengineTask, bind=True) +@app.task(name='dalfox_xss_scan', queue='main_scan_queue', base=RengineTask, bind=True) def dalfox_xss_scan(self, urls=[], ctx={}, description=None): """XSS Scan using dalfox @@ -2407,7 +2406,7 @@ def dalfox_xss_scan(self, urls=[], ctx={}, description=None): return results -@app.task(name='crlfuzz', queue='crlfuzz_queue', base=RengineTask, bind=True) +@app.task(name='crlfuzz', queue='main_scan_queue', base=RengineTask, bind=True) def crlfuzz(self, urls=[], ctx={}, description=None): """CRLF Fuzzing with CRLFuzz @@ -2528,7 +2527,7 @@ def crlfuzz(self, urls=[], ctx={}, description=None): return results -@app.task(name='s3scanner', queue='s3scanner_queue', base=RengineTask, bind=True) +@app.task(name='s3scanner', queue='main_scan_queue', base=RengineTask, bind=True) def s3scanner(self, ctx={}, description=None): """Bucket Scanner @@ -2560,7 +2559,7 @@ def s3scanner(self, ctx={}, description=None): logger.info(f"s3 bucket added {result['provider']}-{result['name']}-{result['region']}") -@app.task(name='http_crawl', queue='http_crawl_queue', base=RengineTask, bind=True) +@app.task(name='http_crawl', queue='main_scan_queue', base=RengineTask, bind=True) def http_crawl( self, urls=[], diff --git a/web/requirements.txt b/web/requirements.txt index 83e9997f..4aa8356a 100644 --- a/web/requirements.txt +++ b/web/requirements.txt @@ -1,7 +1,7 @@ aiodns==3.0.0 argh==0.26.2 beautifulsoup4==4.9.3 -celery==5.2.7 +celery==5.3.4 degoogle==1.0.1 discord-webhook==0.14.0 Django==3.2.4 @@ -16,28 +16,28 @@ djangorestframework-datatables==0.6.0 dotted-dict==1.1.3 drf-yasg==1.21.3 fuzzywuzzy==0.18.0 -gunicorn==20.1.0 -gevent==23.9.0.post1 +gunicorn==21.2.0 +gevent==23.9.1 humanize==4.3.0 Markdown==3.3.4 metafinder==1.1 netaddr==0.8.0 netlas==0.4.0 -openai==0.27.4 +openai==0.28.0 PyYAML==5.4.1 PySocks==1.7.1 -psycopg2==2.9.1 +psycopg2==2.9.7 pycvesearch==1.2 python-Levenshtein==0.12.2 PyVirtualDisplay==2.2 -redis==3.5.3 -requests==2.25.1 +redis==5.0.0 +requests==2.31.0 scapy==2.4.3 selenium==3.141.0 tldextract==3.1.0 uro==1.0.0 validators==0.18.2 -watchdog==2.1.9 +watchdog==3.0.0 whatportis==0.8.2 weasyprint==53.3 wafw00f==2.2.0 diff --git a/web/startScan/models.py b/web/startScan/models.py index c6bbb780..5ef58ca4 100644 --- a/web/startScan/models.py +++ b/web/startScan/models.py @@ -509,6 +509,9 @@ class Command(models.Model): output = models.TextField(blank=True, null=True) time = models.DateTimeField() + def __str__(self): + return str(self.command) + class Waf(models.Model): id = models.AutoField(primary_key=True)