From 7a31c2bb2d59df0a6c247e00a81e0f8785694b28 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Fri, 7 May 2021 09:52:16 +0530 Subject: [PATCH] Fixed Vulnerability Table, added reference, description --- reNgine/tasks.py | 161 ++++++++++-------- startScan/api/views.py | 8 +- .../migrations/0014_auto_20210507_0252.py | 23 +++ startScan/models.py | 8 +- .../templates/startScan/detail_scan.html | 49 +++--- 5 files changed, 150 insertions(+), 99 deletions(-) create mode 100644 startScan/migrations/0014_auto_20210507_0252.py diff --git a/reNgine/tasks.py b/reNgine/tasks.py index 0af95984..f684a34b 100644 --- a/reNgine/tasks.py +++ b/reNgine/tasks.py @@ -443,6 +443,9 @@ def http_crawler(task, domain, results_dir, alive_file_location, activity_id): alive_file.close() + # sort and unique alive urls + os.system('sort -u {} -o {}'.format(alive_file_location, alive_file_location)) + def grab_screenshot(task, yaml_configuration, results_dir, activity_id): ''' @@ -760,10 +763,13 @@ def vulnerability_scan( vulnerability_result_path = results_dir + '/vulnerability.json' + nuclei_scan_urls = results_dir + '/unfurl_urls.txt' + if(task.scan_type.fetch_url): - nuclei_scan_urls = results_dir + '/unfurl_urls.txt' - else: - nuclei_scan_urls = results_dir + '/alive.txt' + os.system('cat {} >> {}'.format(results_dir + '/unfurl_urls.txt', results_dir + '/alive.txt')) + os.system('sort -u {} -o {}'.format(results_dir + '/alive.txt', results_dir + '/alive.txt')) + + nuclei_scan_urls = results_dir + '/alive.txt' nuclei_command = 'nuclei -json -l {} -o {}'.format( nuclei_scan_urls, vulnerability_result_path) @@ -806,82 +812,93 @@ def vulnerability_scan( else: severity = yaml_configuration['vulnerability_scan']['severity'].replace( " ", "") - - # Update nuclei command based on severity - nuclei_command = nuclei_command + ' -severity ' + severity + else: + severity = "critical, high, medium, low, info" # update nuclei templates before running scan os.system('nuclei -update-templates') - # run nuclei - print(nuclei_command) - os.system(nuclei_command) - - try: + for _severity in severity.split(","): + # delete any existing vulnerability.json file if os.path.isfile(vulnerability_result_path): - urls_json_result = open(vulnerability_result_path, 'r') - lines = urls_json_result.readlines() - for line in lines: - json_st = json.loads(line.strip()) - host = json_st['host'] - extracted_subdomain = tldextract.extract(host) - _subdomain = '.'.join(extracted_subdomain[:4]) - if _subdomain[0] == '.': - _subdomain = _subdomain[1:] - try: - subdomain = Subdomain.objects.get( - name=_subdomain, scan_history=task) - vulnerability = Vulnerability() - vulnerability.subdomain = subdomain - vulnerability.scan_history = task - vulnerability.target_domain = domain + os.system('rm {}'.format(vulnerability_result_path)) + # run nuclei + final_nuclei_command = nuclei_command + ' -severity ' + _severity + logger.info(final_nuclei_command) + + os.system(final_nuclei_command) + try: + if os.path.isfile(vulnerability_result_path): + urls_json_result = open(vulnerability_result_path, 'r') + lines = urls_json_result.readlines() + for line in lines: + json_st = json.loads(line.strip()) + host = json_st['host'] + extracted_subdomain = tldextract.extract(host) + _subdomain = '.'.join(extracted_subdomain[:4]) + if _subdomain[0] == '.': + _subdomain = _subdomain[1:] try: - endpoint = EndPoint.objects.get( - scan_history=task, target_domain=domain, http_url=host) - vulnerability.endpoint = endpoint - except Exception as exception: - pass - if 'name' in json_st['info']: - vulnerability.name = json_st['info']['name'] - if 'severity' in json_st['info']: - if json_st['info']['severity'] == 'info': - severity = 0 - elif json_st['info']['severity'] == 'low': - severity = 1 - elif json_st['info']['severity'] == 'medium': - severity = 2 - elif json_st['info']['severity'] == 'high': - severity = 3 - elif json_st['info']['severity'] == 'critical': - severity = 4 + subdomain = Subdomain.objects.get( + name=_subdomain, scan_history=task) + vulnerability = Vulnerability() + vulnerability.subdomain = subdomain + vulnerability.scan_history = task + vulnerability.target_domain = domain + try: + endpoint = EndPoint.objects.get( + scan_history=task, target_domain=domain, http_url=host) + vulnerability.endpoint = endpoint + except Exception as exception: + pass + if 'name' in json_st['info']: + vulnerability.name = json_st['info']['name'] + if 'severity' in json_st['info']: + if json_st['info']['severity'] == 'info': + severity = 0 + elif json_st['info']['severity'] == 'low': + severity = 1 + elif json_st['info']['severity'] == 'medium': + severity = 2 + elif json_st['info']['severity'] == 'high': + severity = 3 + elif json_st['info']['severity'] == 'critical': + severity = 4 + else: + severity = 0 else: severity = 0 - else: - severity = 0 - vulnerability.severity = severity - if 'matched' in json_st: - vulnerability.http_url = json_st['matched'] - if 'templateID' in json_st: - vulnerability.template_used = json_st['templateID'] - if 'description' in json_st: - vulnerability.description = json_st['description'] - if 'matcher_name' in json_st: - vulnerability.matcher_name = json_st['matcher_name'] - if 'extracted_results' in json_st: - vulnerability.extracted_results = json_st['extracted_results'] - vulnerability.discovered_date = timezone.now() - vulnerability.open_status = True - vulnerability.save() - send_notification( - "ALERT! {} vulnerability with {} severity identified in {} \n Vulnerable URL: {}".format( - json_st['info']['name'], - json_st['info']['severity'], - domain.domain_name, - json_st['matched'])) - except ObjectDoesNotExist: - logger.error('Object not found') - except Exception as exception: - logging.error(exception) - update_last_activity(activity_id, 0) + vulnerability.severity = severity + if 'tags' in json_st['info']: + vulnerability.tags = json_st['info']['tags'] + if 'description' in json_st['info']: + vulnerability.description = json_st['info']['description'] + if 'reference' in json_st['info']: + vulnerability.reference = json_st['info']['reference'] + if 'matched' in json_st: + vulnerability.http_url = json_st['matched'] + if 'templateID' in json_st: + vulnerability.template_used = json_st['templateID'] + if 'description' in json_st: + vulnerability.description = json_st['description'] + if 'matcher_name' in json_st: + vulnerability.matcher_name = json_st['matcher_name'] + if 'extracted_results' in json_st: + vulnerability.extracted_results = json_st['extracted_results'] + vulnerability.discovered_date = timezone.now() + vulnerability.open_status = True + vulnerability.save() + send_notification( + "ALERT! {} vulnerability with {} severity identified in {} \n Vulnerable URL: {}".format( + json_st['info']['name'], + json_st['info']['severity'], + domain.domain_name, + json_st['matched'])) + except ObjectDoesNotExist: + logger.error('Object not found') + + except Exception as exception: + logging.error(exception) + update_last_activity(activity_id, 0) def send_notification(message): diff --git a/startScan/api/views.py b/startScan/api/views.py index 34b9b436..a8616771 100644 --- a/startScan/api/views.py +++ b/startScan/api/views.py @@ -357,21 +357,21 @@ class VulnerabilityViewSet(viewsets.ModelViewSet): def get_queryset(self): req = self.request - vulnerability_of = req.query_params.get('vulnerability_of') + vulnerability_of = req.query_params.get('scan_history') url_query = req.query_params.get('query_param') if url_query: if url_query.isnumeric(): self.queryset = Vulnerability.objects.filter( Q( - vulnerability_of__domain_name__domain_name=url_query) | Q( + scan_history__domain_name__domain_name=url_query) | Q( name=url_query) | Q( id=url_query)) else: self.queryset = Vulnerability.objects.filter( - Q(vulnerability_of__domain_name__domain_name=url_query) | Q(name=url_query)) + Q(scan_history__domain_name__domain_name=url_query) | Q(name=url_query)) elif vulnerability_of: self.queryset = Vulnerability.objects.filter( - vulnerability_of__id=vulnerability_of) + scan_history__id=vulnerability_of) return self.queryset def filter_queryset(self, qs): diff --git a/startScan/migrations/0014_auto_20210507_0252.py b/startScan/migrations/0014_auto_20210507_0252.py new file mode 100644 index 00000000..fb1873ec --- /dev/null +++ b/startScan/migrations/0014_auto_20210507_0252.py @@ -0,0 +1,23 @@ +# Generated by Django 3.1.6 on 2021-05-07 02:52 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('startScan', '0013_auto_20210506_0729'), + ] + + operations = [ + migrations.AddField( + model_name='vulnerability', + name='reference', + field=models.CharField(blank=True, max_length=3000, null=True), + ), + migrations.AddField( + model_name='vulnerability', + name='tags', + field=models.CharField(blank=True, max_length=1000, null=True), + ), + ] diff --git a/startScan/models.py b/startScan/models.py index 1b83f826..6f983300 100644 --- a/startScan/models.py +++ b/startScan/models.py @@ -118,14 +118,16 @@ class Vulnerability(models.Model): subdomain = models.ForeignKey(Subdomain, on_delete=models.CASCADE, null=True, blank=True) endpoint = models.ForeignKey(EndPoint, on_delete=models.CASCADE, blank=True, null=True) target_domain = models.ForeignKey(Domain, on_delete=models.CASCADE, null=True, blank=True) - discovered_date = models.DateTimeField(null=True) - http_url = models.CharField(max_length=8000, null=True) + template_used = models.CharField(max_length=100) name = models.CharField(max_length=400) severity = models.IntegerField() description = models.CharField(max_length=1000, null=True, blank=True) extracted_results = models.CharField(max_length=1000, null=True, blank=True) - template_used = models.CharField(max_length=100) + reference = models.CharField(max_length=3000, null=True, blank=True) + tags = models.CharField(max_length=1000, null=True, blank=True) + http_url = models.CharField(max_length=8000, null=True) matcher_name = models.CharField(max_length=400, null=True, blank=True) + discovered_date = models.DateTimeField(null=True) open_status = models.BooleanField(null=True, blank=True, default=True) def __str__(self): diff --git a/startScan/templates/startScan/detail_scan.html b/startScan/templates/startScan/detail_scan.html index 60e38bf6..1bfa40a4 100644 --- a/startScan/templates/startScan/detail_scan.html +++ b/startScan/templates/startScan/detail_scan.html @@ -388,7 +388,7 @@ Detailed Scan Results for {{history.domain_name}} {% elif vulnerability.severity == 4 %} Critical {% endif %} - {{vulnerability.url|truncatechars:50}} + {{vulnerability.http_url|truncatechars:50}} {% endfor %} @@ -478,7 +478,9 @@ Detailed Scan Results for {{history.domain_name}} Vulnerable URL Detail Status - Last Seen + Matcher Name + Tags + Reference @@ -934,13 +936,16 @@ Detailed Scan Results for {{history.domain_name}} {'data': 'severity'}, {'data': 'http_url'}, {'data': 'extracted_results'}, - {'data': 'discovered_date'}, {'data': 'open_status'}, {'data': 'matcher_name'}, + {'data': 'discovered_date'}, + {'data': 'tags'}, + {'data': 'description'}, + {'data': 'reference'}, ], "columnDefs": [ { - "targets": [ 7 ], + "targets": [ 6, 7, 8, 9, 10 ], "visible": false, "searchable": true, }, @@ -963,25 +968,32 @@ Detailed Scan Results for {{history.domain_name}} }, { "render": function ( data, type, row ) { + var tags_span =""; switch (row['severity']) { case 'Info': - badge = 'text-info' + color = 'info' break; case 'Low': - badge = 'text-low'; + color = 'low' break; case 'Medium': - badge = 'text-warning'; + color = 'warning' break; case 'High': - badge = 'text-danger'; + color = 'danger' break; case 'Critical': - badge = 'text-danger'; + color = 'critical' break; default: } - return ``+data+`
Last Seen: ` + row['discovered_date']+``; + if (row['tags']) { + var badge = ""; + row['tags'].split(/\s*,\s*/).forEach(function(split_vals) { + tags_span+=badge + split_vals + ""; + }); + } + return `` + data + `` + `
` + tags_span + `
Last Seen:` + row['discovered_date']; }, "targets": 1, }, @@ -1009,6 +1021,7 @@ Detailed Scan Results for {{history.domain_name}} }, "targets": 2, }, + { "render": function ( data, type, row ) { return ""+htmlEncode(data)+""; @@ -1017,18 +1030,14 @@ Detailed Scan Results for {{history.domain_name}} }, { "render": function ( data, type, row ) { - extracted_results = row['extracted_results'] ? row['extracted_results'] : "" - matcher_name = row['matcher_name'] ? row['matcher_name'] : "" - return extracted_results + matcher_name; + extracted_results = row['extracted_results'] ? row['extracted_results'] + `
` : ""; + matcher_name = row['matcher_name'] ? row['matcher_name'] + `
` : ""; + description = row['description'] ? row['description'] + `
` : ""; + reference = row['reference'] ? `Read More...` : ""; + return extracted_results + matcher_name + description + reference; }, "targets": 4, }, - { - "render": function ( data, type, row ) { - return htmlEncode(data); - }, - "targets": 5, - }, { "render": function ( data, type, row ) { if (data){ @@ -1038,7 +1047,7 @@ Detailed Scan Results for {{history.domain_name}} return '
' } }, - "targets": 6, + "targets": 5, }, ], drawCallback: function () {