Fixed Vulnerability Table, added reference, description

reNgine/tasks.py

@@ -443,6 +443,9 @@ def http_crawler(task, domain, results_dir, alive_file_location, activity_id):
 
     alive_file.close()
 
+    # sort and unique alive urls
+    os.system('sort -u {} -o {}'.format(alive_file_location, alive_file_location))
+
 
 def grab_screenshot(task, yaml_configuration, results_dir, activity_id):
     '''
@@ -760,10 +763,13 @@ def vulnerability_scan(
 
     vulnerability_result_path = results_dir + '/vulnerability.json'
 
+    nuclei_scan_urls = results_dir + '/unfurl_urls.txt'
+
     if(task.scan_type.fetch_url):
-        nuclei_scan_urls = results_dir + '/unfurl_urls.txt'
-    else:
-        nuclei_scan_urls = results_dir + '/alive.txt'
+        os.system('cat {} >> {}'.format(results_dir + '/unfurl_urls.txt', results_dir + '/alive.txt'))
+        os.system('sort -u {} -o {}'.format(results_dir + '/alive.txt', results_dir + '/alive.txt'))
+
+    nuclei_scan_urls = results_dir + '/alive.txt'
 
     nuclei_command = 'nuclei -json -l {} -o {}'.format(
         nuclei_scan_urls, vulnerability_result_path)
@@ -806,82 +812,93 @@ def vulnerability_scan(
             else:
                 severity = yaml_configuration['vulnerability_scan']['severity'].replace(
                     " ", "")
-
-            # Update nuclei command based on severity
-            nuclei_command = nuclei_command + ' -severity ' + severity
+        else:
+            severity = "critical, high, medium, low, info"
 
     # update nuclei templates before running scan
     os.system('nuclei -update-templates')
-    # run nuclei
-    print(nuclei_command)
-    os.system(nuclei_command)
-
-    try:
+    for _severity in severity.split(","):
+        # delete any existing vulnerability.json file
         if os.path.isfile(vulnerability_result_path):
-            urls_json_result = open(vulnerability_result_path, 'r')
-            lines = urls_json_result.readlines()
-            for line in lines:
-                json_st = json.loads(line.strip())
-                host = json_st['host']
-                extracted_subdomain = tldextract.extract(host)
-                _subdomain = '.'.join(extracted_subdomain[:4])
-                if _subdomain[0] == '.':
-                    _subdomain = _subdomain[1:]
-                try:
-                    subdomain = Subdomain.objects.get(
-                        name=_subdomain, scan_history=task)
-                    vulnerability = Vulnerability()
-                    vulnerability.subdomain = subdomain
-                    vulnerability.scan_history = task
-                    vulnerability.target_domain = domain
+            os.system('rm {}'.format(vulnerability_result_path))
+        # run nuclei
+        final_nuclei_command = nuclei_command + ' -severity ' + _severity
+        logger.info(final_nuclei_command)
+
+        os.system(final_nuclei_command)
+        try:
+            if os.path.isfile(vulnerability_result_path):
+                urls_json_result = open(vulnerability_result_path, 'r')
+                lines = urls_json_result.readlines()
+                for line in lines:
+                    json_st = json.loads(line.strip())
+                    host = json_st['host']
+                    extracted_subdomain = tldextract.extract(host)
+                    _subdomain = '.'.join(extracted_subdomain[:4])
+                    if _subdomain[0] == '.':
+                        _subdomain = _subdomain[1:]
                     try:
-                        endpoint = EndPoint.objects.get(
-                            scan_history=task, target_domain=domain, http_url=host)
-                        vulnerability.endpoint = endpoint
-                    except Exception as exception:
-                        pass
-                    if 'name' in json_st['info']:
-                        vulnerability.name = json_st['info']['name']
-                    if 'severity' in json_st['info']:
-                        if json_st['info']['severity'] == 'info':
-                            severity = 0
-                        elif json_st['info']['severity'] == 'low':
-                            severity = 1
-                        elif json_st['info']['severity'] == 'medium':
-                            severity = 2
-                        elif json_st['info']['severity'] == 'high':
-                            severity = 3
-                        elif json_st['info']['severity'] == 'critical':
-                            severity = 4
+                        subdomain = Subdomain.objects.get(
+                            name=_subdomain, scan_history=task)
+                        vulnerability = Vulnerability()
+                        vulnerability.subdomain = subdomain
+                        vulnerability.scan_history = task
+                        vulnerability.target_domain = domain
+                        try:
+                            endpoint = EndPoint.objects.get(
+                                scan_history=task, target_domain=domain, http_url=host)
+                            vulnerability.endpoint = endpoint
+                        except Exception as exception:
+                            pass
+                        if 'name' in json_st['info']:
+                            vulnerability.name = json_st['info']['name']
+                        if 'severity' in json_st['info']:
+                            if json_st['info']['severity'] == 'info':
+                                severity = 0
+                            elif json_st['info']['severity'] == 'low':
+                                severity = 1
+                            elif json_st['info']['severity'] == 'medium':
+                                severity = 2
+                            elif json_st['info']['severity'] == 'high':
+                                severity = 3
+                            elif json_st['info']['severity'] == 'critical':
+                                severity = 4
+                            else:
+                                severity = 0
                         else:
                             severity = 0
-                    else:
-                        severity = 0
-                    vulnerability.severity = severity
-                    if 'matched' in json_st:
-                        vulnerability.http_url = json_st['matched']
-                    if 'templateID' in json_st:
-                        vulnerability.template_used = json_st['templateID']
-                    if 'description' in json_st:
-                        vulnerability.description = json_st['description']
-                    if 'matcher_name' in json_st:
-                        vulnerability.matcher_name = json_st['matcher_name']
-                    if 'extracted_results' in json_st:
-                        vulnerability.extracted_results = json_st['extracted_results']
-                    vulnerability.discovered_date = timezone.now()
-                    vulnerability.open_status = True
-                    vulnerability.save()
-                    send_notification(
-                        "ALERT! {} vulnerability with {} severity identified in {} \n Vulnerable URL: {}".format(
-                            json_st['info']['name'],
-                            json_st['info']['severity'],
-                            domain.domain_name,
-                            json_st['matched']))
-                except ObjectDoesNotExist:
-                    logger.error('Object not found')
-    except Exception as exception:
-        logging.error(exception)
-        update_last_activity(activity_id, 0)
+                        vulnerability.severity = severity
+                        if 'tags' in json_st['info']:
+                            vulnerability.tags = json_st['info']['tags']
+                        if 'description' in json_st['info']:
+                            vulnerability.description = json_st['info']['description']
+                        if 'reference' in json_st['info']:
+                            vulnerability.reference = json_st['info']['reference']
+                        if 'matched' in json_st:
+                            vulnerability.http_url = json_st['matched']
+                        if 'templateID' in json_st:
+                            vulnerability.template_used = json_st['templateID']
+                        if 'description' in json_st:
+                            vulnerability.description = json_st['description']
+                        if 'matcher_name' in json_st:
+                            vulnerability.matcher_name = json_st['matcher_name']
+                        if 'extracted_results' in json_st:
+                            vulnerability.extracted_results = json_st['extracted_results']
+                        vulnerability.discovered_date = timezone.now()
+                        vulnerability.open_status = True
+                        vulnerability.save()
+                        send_notification(
+                            "ALERT! {} vulnerability with {} severity identified in {} \n Vulnerable URL: {}".format(
+                                json_st['info']['name'],
+                                json_st['info']['severity'],
+                                domain.domain_name,
+                                json_st['matched']))
+                    except ObjectDoesNotExist:
+                        logger.error('Object not found')
+
+        except Exception as exception:
+            logging.error(exception)
+            update_last_activity(activity_id, 0)
 
 
 def send_notification(message):

startScan/api/views.py

@@ -357,21 +357,21 @@ class VulnerabilityViewSet(viewsets.ModelViewSet):
 
     def get_queryset(self):
         req = self.request
-        vulnerability_of = req.query_params.get('vulnerability_of')
+        vulnerability_of = req.query_params.get('scan_history')
         url_query = req.query_params.get('query_param')
         if url_query:
             if url_query.isnumeric():
                 self.queryset = Vulnerability.objects.filter(
                     Q(
-                        vulnerability_of__domain_name__domain_name=url_query) | Q(
+                        scan_history__domain_name__domain_name=url_query) | Q(
                         name=url_query) | Q(
                         id=url_query))
             else:
                 self.queryset = Vulnerability.objects.filter(
-                    Q(vulnerability_of__domain_name__domain_name=url_query) | Q(name=url_query))
+                    Q(scan_history__domain_name__domain_name=url_query) | Q(name=url_query))
         elif vulnerability_of:
             self.queryset = Vulnerability.objects.filter(
-                vulnerability_of__id=vulnerability_of)
+                scan_history__id=vulnerability_of)
         return self.queryset
 
     def filter_queryset(self, qs):

startScan/migrations/0014_auto_20210507_0252.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.1.6 on 2021-05-07 02:52
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('startScan', '0013_auto_20210506_0729'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='vulnerability',
+            name='reference',
+            field=models.CharField(blank=True, max_length=3000, null=True),
+        ),
+        migrations.AddField(
+            model_name='vulnerability',
+            name='tags',
+            field=models.CharField(blank=True, max_length=1000, null=True),
+        ),
+    ]

startScan/models.py

@@ -118,14 +118,16 @@ class Vulnerability(models.Model):
     subdomain = models.ForeignKey(Subdomain, on_delete=models.CASCADE, null=True, blank=True)
     endpoint = models.ForeignKey(EndPoint, on_delete=models.CASCADE, blank=True, null=True)
     target_domain = models.ForeignKey(Domain, on_delete=models.CASCADE, null=True, blank=True)
-    discovered_date = models.DateTimeField(null=True)
-    http_url = models.CharField(max_length=8000, null=True)
+    template_used = models.CharField(max_length=100)
     name = models.CharField(max_length=400)
     severity = models.IntegerField()
     description = models.CharField(max_length=1000, null=True, blank=True)
     extracted_results = models.CharField(max_length=1000, null=True, blank=True)
-    template_used = models.CharField(max_length=100)
+    reference = models.CharField(max_length=3000, null=True, blank=True)
+    tags = models.CharField(max_length=1000, null=True, blank=True)
+    http_url = models.CharField(max_length=8000, null=True)
     matcher_name = models.CharField(max_length=400, null=True, blank=True)
+    discovered_date = models.DateTimeField(null=True)
     open_status = models.BooleanField(null=True, blank=True, default=True)
 
     def __str__(self):

startScan/templates/startScan/detail_scan.html

@@ -388,7 +388,7 @@ Detailed Scan Results for {{history.domain_name}}
 															{% elif vulnerability.severity == 4 %}
 															<span class='badge badge-critical'>Critical</span>
 															{% endif %}</td>
-															<td><a href="{{vulnerability.url}}" target="_blank" class="text-danger bs-tooltip" title="{{vulnerability.url}}">{{vulnerability.url|truncatechars:50}}</a></td>
+															<td><a href="{{vulnerability.http_url}}" target="_blank" class="text-danger bs-tooltip" title="{{vulnerability.http_url}}">{{vulnerability.http_url|truncatechars:50}}</a></td>
 														</tr>
 														{% endfor %}
 													</tbody>
@@ -478,7 +478,9 @@ Detailed Scan Results for {{history.domain_name}}
 										<th>Vulnerable URL</th>
 										<th>Detail</th>
 										<th>Status</th>
-										<th>Last Seen</th>
+										<th>Matcher Name</th>
+										<th>Tags</th>
+										<th>Reference</th>
 									</tr>
 								</thead>
 							</table>
@@ -934,13 +936,16 @@ Detailed Scan Results for {{history.domain_name}}
 				{'data': 'severity'},
 				{'data': 'http_url'},
 				{'data': 'extracted_results'},
-				{'data': 'discovered_date'},
 				{'data': 'open_status'},
 				{'data': 'matcher_name'},
+				{'data': 'discovered_date'},
+				{'data': 'tags'},
+				{'data': 'description'},
+				{'data': 'reference'},
 			],
 			"columnDefs": [
 				{
-					"targets": [ 7 ],
+					"targets": [ 6, 7, 8, 9, 10 ],
 					"visible": false,
 					"searchable": true,
 				},
@@ -963,25 +968,32 @@ Detailed Scan Results for {{history.domain_name}}
 				},
 				{
 					"render": function ( data, type, row ) {
+						var tags_span ="";
 						switch (row['severity']) {
 							case 'Info':
-							badge = 'text-info'
+							color = 'info'
 							break;
 							case 'Low':
-							badge = 'text-low';
+							color = 'low'
 							break;
 							case 'Medium':
-							badge = 'text-warning';
+							color = 'warning'
 							break;
 							case 'High':
-							badge = 'text-danger';
+							color = 'danger'
 							break;
 							case 'Critical':
-							badge = 'text-danger';
+							color = 'critical'
 							break;
 							default:
 						}
-						return `<b class="${badge}">`+data+`</b><br><small>Last Seen: ` + row['discovered_date']+`</small>`;
+						if (row['tags']) {
+							var badge = "<span class='badge badge-pill badge-"+color+" m-1'>";
+							row['tags'].split(/\s*,\s*/).forEach(function(split_vals) {
+								tags_span+=badge + split_vals + "</span>";
+							});
+						}
+						return `<b class="text-${color}">` + data + `</b>` + `</small><br/>` + tags_span + `<br><small>Last Seen:` + row['discovered_date'];
 					},
 					"targets": 1,
 				},
@@ -1009,6 +1021,7 @@ Detailed Scan Results for {{history.domain_name}}
 					},
 					"targets": 2,
 				},
+
 				{
 					"render": function ( data, type, row ) {
 						return "<a href='"+htmlEncode(data)+"' target='_blank' class='text-danger'>"+htmlEncode(data)+"</a>";
@@ -1017,18 +1030,14 @@ Detailed Scan Results for {{history.domain_name}}
 				},
 				{
 					"render": function ( data, type, row ) {
-						extracted_results = row['extracted_results'] ? row['extracted_results'] : ""
-						matcher_name = row['matcher_name'] ? row['matcher_name'] : ""
-						return extracted_results + matcher_name;
+						extracted_results = row['extracted_results'] ? row['extracted_results'] + `</br>` : "";
+						matcher_name = row['matcher_name'] ? row['matcher_name'] + `</br>` : "";
+						description = row['description'] ? row['description'] + `</br>` : "";
+						reference = row['reference'] ? `<a href=${row['reference']} class='text-info' target='_blank'>Read More...</a>` : "";
+						return extracted_results + matcher_name + description + reference;
 					},
 					"targets": 4,
 				},
-				{
-					"render": function ( data, type, row ) {
-						return htmlEncode(data);
-					},
-					"targets": 5,
-				},
 				{
 					"render": function ( data, type, row ) {
 						if (data){
@@ -1038,7 +1047,7 @@ Detailed Scan Results for {{history.domain_name}}
 							return '<div class="t-dot bg-success" data-toggle="tooltip" data-placement="top" title="" data-original-title="Closed"></div>'
 						}
 					},
-					"targets": 6,
+					"targets": 5,
 				},
 			],
 			drawCallback: function () {