gau fetches all urls

startScan/admin.py

@@ -1,6 +1,7 @@
 from django.contrib import admin
-from startScan.models import ScanHistory, ScannedHost, ScanActivity
+from startScan.models import ScanHistory, ScannedHost, ScanActivity, WayBackEndPoint
 
 admin.site.register(ScanHistory)
 admin.site.register(ScannedHost)
 admin.site.register(ScanActivity)
+admin.site.register(WayBackEndPoint)

startScan/migrations/0017_waybackendpoint.py

@@ -0,0 +1,25 @@
+# Generated by Django 3.0.6 on 2020-05-31 21:05
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('startScan', '0016_auto_20200531_2345'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='WayBackEndPoint',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('http_url', models.CharField(max_length=1000)),
+                ('content_length', models.IntegerField(default=0)),
+                ('page_title', models.CharField(max_length=1000)),
+                ('http_status', models.IntegerField(default=0)),
+                ('url_of', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='startScan.ScanHistory')),
+            ],
+        ),
+    ]

startScan/templates/startScan/detail_scan.html

@@ -77,7 +77,7 @@ Scan history
                                 <svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg>
                             </div>
                             <br>
-                            <h2 class="text-warning">600</h2>
+                            <h2 class="text-warning">{{endpoints.count}}</h2>
                             <h6 class="">Endpoints</h6>
                         </div>
                         <div class="widget-content">
@@ -209,12 +209,18 @@ Scan history
                         </tr>
                     </thead>
                     <tbody>
-                        <tr>
-                            <td>Hello</td>
-                            <td>Hello</td>
-                            <td>Hello</td>
-                            <td>Hello</td>
-                        </tr>
+                        {% for endpoint in  endpoints%}
+                            <tr>
+                                <td><a href="{{endpoint.http_url}}" target="_blank">{{endpoint.http_url}}</a></td>
+                                <td>
+                                    {% if endpoint.http_status > 0 %}
+                                        <span class="badge badge-pills {% if endpoint.http_status >= 200 and endpoint.http_status < 300 %}badge-success {% elif endpoint.http_status >= 300 and endpoint.http_status < 400 %} badge-warning {% elif endpoint.http_status >= 400 and endpoint.http_status < 500 %}badge-danger {% else %}badge-danger{% endif %}">{{endpoint.http_status}}</span>
+                                    {% endif %}
+                                </td>
+                                <td>{{endpoint.content_length}}</td>
+                                <td>{{endpoint.page_title}}</td>
+                            </tr>
+                        {% endfor %}
                     </tbody>
                 </table>
             </div>

startScan/views.py

@@ -2,7 +2,7 @@ from django.shortcuts import render, get_object_or_404
 from django.contrib import messages
 from django.http import JsonResponse, HttpResponseRedirect
 from django.urls import reverse
-from .models import ScanHistory, ScannedHost, ScanActivity
+from .models import ScanHistory, ScannedHost, ScanActivity, WayBackEndPoint
 from notification.models import NotificationHooks
 from targetApp.models import Domain
 from scanEngine.models import EngineType
@@ -25,11 +25,13 @@ def detail_scan(request, id):
     subdomain_details = ScannedHost.objects.filter(scan_history__id=id)
     alive_count = ScannedHost.objects.filter(scan_history__id=id).exclude(http_status__exact=0).count()
     scan_activity = ScanActivity.objects.filter(scan_of__id=id)
+    endpoints = WayBackEndPoint.objects.filter(url_of__id=id)
     context = {'scan_history_active': 'true',
                 'subdomain':subdomain_details,
                 'scan_history':scan_history,
                 'scan_activity':scan_activity,
-                'alive_count':alive_count
+                'alive_count':alive_count,
+                'endpoints':endpoints
                 }
     return render(request, 'startScan/detail_scan.html', context)
 
@@ -173,9 +175,23 @@ def doScan(id, domain):
     save_scan_activity(task, "Fetching endpoints", 1)
     wayback_results_file = results_dir + current_scan_dir + '/wayback.json'
 
-    wayback_command = 'echo ' + domain.domain_name + ' | /app/tools/gau | /app/tools/httpx -status-code -content-length -title -json -o {}'.format(wayback_results_file)
+    wayback_command = 'echo ' + domain.domain_name + ' | /app/tools/gau -providers wayback | /app/tools/httpx -status-code -content-length -title -json -o {}'.format(wayback_results_file)
     os.system(wayback_command)
 
+    wayback_json_result = open(wayback_results_file, 'r')
+    lines = wayback_json_result.readlines()
+    for line in lines:
+        json_st = json.loads(line.strip())
+        endpoint = WayBackEndPoint()
+        endpoint.url_of = task
+        endpoint.http_url = json_st['url']
+        endpoint.content_length = json_st['content-length']
+        endpoint.http_status = json_st['status-code']
+        endpoint.page_title = json_st['title']
+        endpoint.save()
+
+
+
     task.scan_status = 2
     task.save()