diff --git a/web/Dockerfile b/web/Dockerfile index 001142b2..972bfe98 100644 --- a/web/Dockerfile +++ b/web/Dockerfile @@ -40,7 +40,8 @@ RUN apt install -y --no-install-recommends \ nmap \ x11-utils \ xvfb \ - wget + wget \ + curl # Download and install go 1.20 RUN wget https://golang.org/dl/go1.20.linux-amd64.tar.gz diff --git a/web/celery-entrypoint.sh b/web/celery-entrypoint.sh index 89db0401..d7e48d76 100755 --- a/web/celery-entrypoint.sh +++ b/web/celery-entrypoint.sh @@ -123,6 +123,14 @@ then git clone https://github.com/UnaPibaGeek/ctfr /usr/src/github/ctfr fi +# clone gooFuzz +if [ ! -d "/usr/src/github/goofuzz" ] +then + echo "Cloning GooFuzz" + git clone https://github.com/m3n0sd0n4ld/GooFuzz.git /usr/src/github/goofuzz + chmod +x /usr/src/github/goofuzz/GooFuzz +fi + exec "$@" # httpx seems to have issue, use alias instead!!! @@ -147,5 +155,6 @@ watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/re watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=50 --loglevel=info -Q run_command_queue -n run_command_worker & watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=info -Q query_reverse_whois_queue -n query_reverse_whois_worker & watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=info -Q query_ip_history_queue -n query_ip_history_worker & -watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=30 --loglevel=info -Q gpt_queue -n gpt_worker +watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=30 --loglevel=info -Q gpt_queue -n gpt_worker & +watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=info -Q dorking_queue -n dorking_worker exec "$@" diff --git a/web/reNgine/definitions.py b/web/reNgine/definitions.py index de4844bf..3a848111 100644 --- a/web/reNgine/definitions.py +++ b/web/reNgine/definitions.py @@ -475,3 +475,7 @@ ATTACK_SUGGESTION_GPT_SYSTEM_PROMPT = """ Please provide a detailed list of these attack types and their underlying technical rationales on every attacks you suggested. Also suggest if any CVE ID, known exploits, existing vulnerabilities, any news articles URL related to the information provided to you. """ + + +# OSINT GooFuzz Path +GOFUZZ_EXEC_PATH = '/usr/src/github/goofuzz/GooFuzz' diff --git a/web/reNgine/tasks.py b/web/reNgine/tasks.py index 9f4fe600..7aa6f184 100644 --- a/web/reNgine/tasks.py +++ b/web/reNgine/tasks.py @@ -17,7 +17,6 @@ from api.serializers import SubdomainSerializer from celery import chain, chord, group from celery.result import allow_join_result from celery.utils.log import get_task_logger -from degoogle import degoogle from django.db.models import Count from dotted_dict import DottedDict from django.utils import timezone @@ -589,18 +588,33 @@ def osint(self, host=None, ctx={}, description=None): config = self.yaml_configuration.get(OSINT) or OSINT_DEFAULT_CONFIG results = {} - if 'discover' in config: - ctx['track'] = False - results = osint_discovery(host=host, ctx=ctx) - - # if 'dork' in config: + grouped_tasks = [] + # + # if 'discover' in config: # ctx['track'] = False - # results['dorks'] = dorking(host=host, ctx=ctx) + # results = osint_discovery(host=host, ctx=ctx) - with open(self.output_path, 'w') as f: - json.dump(results, f, indent=4) + if 'dork' in config: + _task = dorking.si( + config=config, + host=self.scan.domain.name, + scan_history_id=self.scan.id, + results_dir=self.results_dir + ) + grouped_tasks.append(_task) - return results + celery_group = group(grouped_tasks) + job = celery_group.apply_async() + while not job.ready(): + # wait for all jobs to complete + time.sleep(5) + + logger.info('OSINT Tasks finished...') + + # with open(self.output_path, 'w') as f: + # json.dump(results, f, indent=4) + # + # return results @app.task(name='osint_discovery', queue='main_scan_queue', base=RengineTask, bind=True) @@ -663,295 +677,217 @@ def osint_discovery(self, host=None, ctx={}): return results -@app.task(name='dorking', queue='main_scan_queue', base=RengineTask, bind=True) -def dorking(self, host=None, ctx={}): +@app.task(name='dorking', bind=False, queue='dorking_queue') +def dorking(config, host, scan_history_id, results_dir): """Run Google dorks. Args: - host (str): Hostname to scan. + config (dict): yaml_configuration + host (str): target name + scan_history_id (startScan.ScanHistory): Scan History ID + results_dir (str): Path to store scan results Returns: list: Dorking results for each dork ran. """ # Some dork sources: https://github.com/six2dez/degoogle_hunter/blob/master/degoogle_hunter.sh - config = self.yaml_configuration.get(OSINT) or OSINT_DEFAULT_CONFIG + scan_history = ScanHistory.objects.get(pk=scan_history_id) dorks = config.get(OSINT_DORK, []) results = [] - for dork in dorks: - if dork == 'stackoverflow': - dork_name = 'site:stackoverflow.com' - dork_type = 'stackoverflow' - results = get_and_save_dork_results( - dork, - dork_type, - host=host, - scan_history=self.scan, - in_target=False) + try: + for dork in dorks: + logger.info(f'Getting dork information for {dork}') + if dork == 'stackoverflow': + results = get_and_save_dork_results( + lookup_target='stackoverflow.com', + results_dir=results_dir, + type=dork, + lookup_keywords=host, + scan_history=scan_history + ) - elif dork == '3rdparty' : - # look in 3rd party sitee - dork_type = '3rdparty' - lookup_websites = [ - 'gitter.im', - 'papaly.com', - 'productforums.google.com', - 'coggle.it', - 'replt.it', - 'ycombinator.com', - 'libraries.io', - 'npm.runkit.com', - 'npmjs.com', - 'scribd.com', - 'gitter.im' - ] - dork_name = '' - for website in lookup_websites: - dork_name = dork + ' | ' + 'site:' + website - tmp_results = get_and_save_dork_results( - dork_name[3:], - dork_type, - host=host, - scan_history=self.scan, - in_target=False) - results.extend(tmp_results) + elif dork == 'social_media' : + social_websites = [ + 'tiktok.com', + 'facebook.com', + 'twitter.com', + 'youtube.com', + 'pinterest.com', + 'tumblr.com', + 'reddit.com' + ] + for site in social_websites: + results = get_and_save_dork_results( + lookup_target=site, + results_dir=results_dir, + type=dork, + lookup_keywords=host, + scan_history=scan_history + ) - elif dork == 'social_media' : - dork_type = 'Social Media' - social_websites = [ - 'tiktok.com', - 'facebook.com', - 'twitter.com', - 'youtube.com', - 'pinterest.com', - 'tumblr.com', - 'reddit.com' - ] - dork_name = '' - for website in social_websites: - dork_name = dork + ' | ' + 'site:' + website - tmp_results = get_and_save_dork_results( - dork_name[3:], - dork_type, - host=host, - scan_history=self.scan, - in_target=False) - results.extend(tmp_results) + elif dork == 'project_management' : + project_websites = [ + 'trello.com', + 'atlassian.net' + ] + for site in project_websites: + results = get_and_save_dork_results( + lookup_target=site, + results_dir=results_dir, + type=dork, + lookup_keywords=host, + scan_history=scan_history + ) - elif dork == 'project_management' : - dork_type = 'Project Management' - project_websites = [ - 'trello.com', - '*.atlassian.net' - ] - dork_name = '' - for website in project_websites: - dork_name = dork + ' | ' + 'site:' + website - tmp_results = get_and_save_dork_results( - dork_name[3:], - dork_type, - host=host, - scan_history=self.scan, - in_target=False) - results.extend(tmp_results) + elif dork == 'code_sharing' : + project_websites = [ + 'github.com', + 'gitlab.com', + 'bitbucket.org' + ] + for site in project_websites: + results = get_and_save_dork_results( + lookup_target=site, + results_dir=results_dir, + type=dork, + lookup_keywords=host, + scan_history=scan_history + ) - elif dork == 'code_sharing' : - dork_type = 'Code Sharing Sites' - code_websites = [ - 'github.com', - 'gitlab.com', - 'bitbucket.org' - ] - dork_name = '' - for website in code_websites: - dork_name = dork + ' | ' + 'site:' + website - tmp_results = get_and_save_dork_results( - dork_name[3:], - dork_type, - host=host, - scan_history=self.scan, - in_target=False) - results.extend(tmp_results) + elif dork == 'config_files' : + config_file_exts = [ + 'env', + 'xml', + 'conf', + 'toml', + 'yml', + 'yaml', + 'cnf', + 'inf', + 'rdp', + 'ora', + 'txt', + 'cfg', + 'ini' + ] + results = get_and_save_dork_results( + lookup_target=site, + results_dir=results_dir, + type=dork, + lookup_extensions=','.join(config_file_exts), + page_count=4, + scan_history=scan_history + ) - elif dork == 'config_files' : - dork_type = 'Config Files' - config_file_ext = [ - 'env', - 'xml', - 'conf', - 'cnf', - 'inf', - 'rdp', - 'ora', - 'txt', - 'cfg', - 'ini' - ] + elif dork == 'jenkins' : + lookup_keyword = 'Jenkins' + results = get_and_save_dork_results( + lookup_target=site, + results_dir=results_dir, + type=dork, + lookup_keywords=lookup_keyword, + page_count=1, + scan_history=scan_history + ) - dork_name = '' - results = [] - for extension in config_file_ext: - dork_name = dork + ' | ' + 'ext:' + extension - tmp_results = get_and_save_dork_results( - dork_name[3:], - dork_type, - host=host, - scan_history=self.scan, - in_target=True) - results.extend(tmp_results) + elif dork == 'wordpress_files' : + lookup_keywords = [ + '/wp-content/', + '/wp-includes/' + ] + results = get_and_save_dork_results( + lookup_target=site, + results_dir=results_dir, + type=dork, + lookup_keywords=','.join(lookup_keywords), + page_count=5, + scan_history=scan_history + ) - if dork == 'jenkins' : - dork_type = 'Jenkins' - dork_name = 'intitle:\"Dashboard [Jenkins]\"' - tmp_results = get_and_save_dork_results( - dork_name, - dork_type, - host=host, - scan_history=self.scan, - in_target=True) - results.extend(tmp_results) + elif dork == 'php_error' : + lookup_keywords = [ + 'PHP Parse error', + 'PHP Warning', + 'PHP Error' + ] + results = get_and_save_dork_results( + lookup_target=site, + results_dir=results_dir, + type=dork, + lookup_keywords=','.join(lookup_keywords), + page_count=5, + scan_history=scan_history + ) - elif dork == 'wordpress_files' : - dork_type = 'Wordpress Files' - inurl_lookup = [ - 'wp-content', - 'wp-includes' - ] - dork_name = '' - for lookup in inurl_lookup: - dork_name = dork + ' | ' + 'inurl:' + lookup - tmp_results = get_and_save_dork_results( - dork_name[3:], - dork_type, - host=host, - scan_history=self.scan, - in_target=True) - results.extend(tmp_results) + elif dork == 'jenkins' : + lookup_keywords = [ + 'PHP Parse error', + 'PHP Warning', + 'PHP Error' + ] + results = get_and_save_dork_results( + lookup_target=site, + results_dir=results_dir, + type=dork, + lookup_keywords=','.join(lookup_keywords), + page_count=5, + scan_history=scan_history + ) - elif dork == 'cloud_buckets': - dork_type = 'Cloud Buckets' - cloud_websites = [ - '.s3.amazonaws.com', - 'storage.googleapis.com', - 'amazonaws.com' - ] + elif dork == 'exposed_documents' : + docs_file_ext = [ + 'doc', + 'docx', + 'odt', + 'pdf', + 'rtf', + 'sxw', + 'psw', + 'ppt', + 'pptx', + 'pps', + 'csv' + ] + results = get_and_save_dork_results( + lookup_target=site, + results_dir=results_dir, + type=dork, + lookup_extensions=','.join(docs_file_ext), + page_count=7, + scan_history=scan_history + ) - dork_name = '' - for website in cloud_websites: - dork_name = dork + ' | ' + 'site:' + website - tmp_results = get_and_save_dork_results( - dork_name[3:], - dork_type, - host=host, - scan_history=self.scan, - in_target=False) - results.extend(tmp_results) + elif dork == 'db_files' : + file_ext = [ + 'sql', + 'db', + 'dbf', + 'mdb' + ] + results = get_and_save_dork_results( + lookup_target=site, + results_dir=results_dir, + type=dork, + lookup_extensions=','.join(file_ext), + page_count=1, + scan_history=scan_history + ) - elif dork == 'php_error': - dork_type = 'PHP Error' - error_words = [ - '\"PHP Parse error\"', - '\"PHP Warning\"', - '\"PHP Error\"' - ] + elif dork == 'git_exposed' : + file_ext = [ + 'git', + ] + results = get_and_save_dork_results( + lookup_target=site, + results_dir=results_dir, + type=dork, + lookup_extensions=','.join(file_ext), + page_count=1, + scan_history=scan_history + ) - dork_name = '' - for word in error_words: - dork_name = dork + ' | ' + word - tmp_results = get_and_save_dork_results( - dork_name[3:], - dork_type, - host=host, - scan_history=self.scan, - in_target=True) - results.extend(tmp_results) - - elif dork == 'exposed_documents': - dork_type = 'Exposed Documents' - docs_file_ext = [ - 'doc', - 'docx', - 'odt', - 'pdf', - 'rtf', - 'sxw', - 'psw', - 'ppt', - 'pptx', - 'pps', - 'csv' - ] - - dork_name = '' - for extension in docs_file_ext: - dork_name = dork + ' | ' + 'ext:' + extension - tmp_results = get_and_save_dork_results( - dork_name[3:], - dork_type, - host=host, - scan_history=self.scan, - in_target=True) - results.extend(tmp_results) - - elif dork == 'struts_rce': - dork_type = 'Apache Struts RCE' - struts_file_ext = [ - 'action', - 'struts', - 'do' - ] - - dork_name = '' - for extension in struts_file_ext: - dork_name = dork + ' | ' + 'ext:' + extension - tmp_results = get_and_save_dork_results( - dork_name[3:], - dork_type, - host=host, - scan_history=self.scan, - in_target=True) - results.extend(tmp_results) - - elif dork == 'db_files': - dork_type = 'Database Files' - db_file_ext = [ - 'sql', - 'db', - 'dbf', - 'mdb' - ] - - dork_name = '' - for extension in db_file_ext: - dork_name = dork_name + ' | ' + 'ext:' + extension - tmp_results = get_and_save_dork_results( - dork_name[3:], - dork_type, - host=host, - scan_history=self.scan, - in_target=True) - results.extend(tmp_results) - - elif dork == 'traefik': - dork_name = 'intitle:traefik inurl:8080/dashboard' - dork_type = 'Traefik' - tmp_results = get_and_save_dork_results( - dork_name, - dork_type, - host=host, - scan_history=self.scan, - in_target=True) - results.extend(tmp_results) - - elif dork == 'git_exposed': - dork_name = 'inurl:\"/.git\"' - dork_type = '.git Exposed' - tmp_results = get_and_save_dork_results( - dork_name, - dork_type, - host=host, - scan_history=self.scan, - in_target=True) - results.extend(tmp_results) + except Exception as e: + logger.exception(e) return results @@ -4127,32 +4063,58 @@ def extract_httpx_url(line): # OSInt utils # #-------------# -def get_and_save_dork_results(dork, type, host=None, scan_history=None, in_target=False): - degoogle_obj = degoogle.dg() - get_random_proxy() - host = scan_history.domain.name if scan_history else host - if in_target: - query = f'{dork} site:{host}' - else: - query = f'{dork} \"{host}\"' - degoogle_obj.query = query - logger.info(f'Running degoogle with query "{query}" ...') - results = degoogle_obj.run() - dorks = [] - if not results: - logger.warning('No data recovered from degoogle.') - return [] - for result in results: - dork, created = Dork.objects.get_or_create( - type=type, - description=result['desc'], - url=result['url'] +def get_and_save_dork_results(lookup_target, results_dir, type, lookup_keywords=None, lookup_extensions=None, delay=3, page_count=2, scan_history=None): + """ + Uses gofuzz to dork and store information + + Args: + lookup_target (str): target to look into such as stackoverflow or even the target itself + results_dir (str): Results directory + type (str): Dork Type Title + lookup_keywords (str): comma separated keywords or paths to look for + lookup_extensions (str): comma separated extensions to look for + delay (int): delay between each requests + page_count (int): pages in google to extract information + scan_history (startScan.ScanHistory): Scan History Object + """ + results = [] + gofuzz_command = f'{GOFUZZ_EXEC_PATH} -t {lookup_target} -d {delay} -p {page_count}' + + if lookup_extensions: + gofuzz_command += f' -e {lookup_extensions}' + elif lookup_keywords: + gofuzz_command += f' -w {lookup_keywords}' + + output_file = f'{results_dir}/gofuzz.txt' + gofuzz_command += f' -o {output_file}' + history_file = f'{results_dir}/commands.txt' + + try: + run_command( + gofuzz_command, + shell=False, + history_file=history_file, + scan_id=scan_history.id, ) - if created: - logger.warning(f'Found dork {dork}') - if scan_history: - scan_history.dorks.add(dork) - dorks.append(dork) + + if not os.path.isfile(output_file): + return + + with open(output_file) as f: + for line in f.readlines(): + url = line.strip() + if url: + results.append(url) + dork, created = Dork.objects.get_or_create( + type=type, + url=url + ) + if scan_history: + scan_history.dorks.add(dork) + + except Exception as e: + logger.exception(e) + return results diff --git a/web/requirements.txt b/web/requirements.txt index 9b19b1c2..49966c96 100644 --- a/web/requirements.txt +++ b/web/requirements.txt @@ -2,7 +2,6 @@ aiodns==3.0.0 argh==0.26.2 beautifulsoup4==4.9.3 celery==5.3.4 -degoogle==1.0.1 discord-webhook==1.3.0 Django==3.2.4 django-ace==1.0.11 diff --git a/web/startScan/migrations/0055_remove_dork_description.py b/web/startScan/migrations/0055_remove_dork_description.py new file mode 100644 index 00000000..1cc5e8bd --- /dev/null +++ b/web/startScan/migrations/0055_remove_dork_description.py @@ -0,0 +1,17 @@ +# Generated by Django 3.2.4 on 2023-09-22 08:57 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('startScan', '0054_auto_20230910_0533'), + ] + + operations = [ + migrations.RemoveField( + model_name='dork', + name='description', + ), + ] diff --git a/web/startScan/models.py b/web/startScan/models.py index 5ef58ca4..5423a28a 100644 --- a/web/startScan/models.py +++ b/web/startScan/models.py @@ -626,7 +626,6 @@ class Employee(models.Model): class Dork(models.Model): id = models.AutoField(primary_key=True) type = models.CharField(max_length=500, null=True, blank=True) - description = models.CharField(max_length=1500, null=True, blank=True) url = models.CharField(max_length=10000, null=True, blank=True)