diff --git a/web/reNgine/celery_custom_task.py b/web/reNgine/celery_custom_task.py
index 2c6b11f2..863f7716 100644
--- a/web/reNgine/celery_custom_task.py
+++ b/web/reNgine/celery_custom_task.py
@@ -67,7 +67,7 @@ class RengineTask(Task):
self.subscan_id = ctx.get('subscan_id')
self.engine_id = ctx.get('engine_id')
self.filename = ctx.get('filename')
- self.starting_point_url = ctx.get('starting_point_url', '')
+ self.starting_point_path = ctx.get('starting_point_path', '')
self.excluded_paths = ctx.get('excluded_paths', [])
self.results_dir = ctx.get('results_dir', RENGINE_RESULTS)
self.yaml_configuration = ctx.get('yaml_configuration', {})
diff --git a/web/reNgine/tasks.py b/web/reNgine/tasks.py
index 33ecbae0..d23086c9 100644
--- a/web/reNgine/tasks.py
+++ b/web/reNgine/tasks.py
@@ -57,7 +57,7 @@ def initiate_scan(
imported_subdomains=[],
out_of_scope_subdomains=[],
initiated_by_id=None,
- starting_point_url='',
+ starting_point_path='',
excluded_paths=[],
):
"""Initiate a new scan.
@@ -70,7 +70,7 @@ def initiate_scan(
results_dir (str): Results directory.
imported_subdomains (list): Imported subdomains.
out_of_scope_subdomains (list): Out-of-scope subdomains.
- starting_point_url (str): URL path. Default: '' Defined where to start the scan.
+ starting_point_path (str): URL path. Default: '' Defined where to start the scan.
initiated_by (int): User ID initiating the scan.
excluded_paths (list): Excluded paths. Default: [], url paths to exclude from scan.
"""
@@ -92,7 +92,7 @@ def initiate_scan(
domain.save()
# Get path filter
- starting_point_url = starting_point_url.rstrip('/')
+ starting_point_path = starting_point_path.rstrip('/')
# for live scan scan history id is passed as scan_history_id
# and no need to create scan_history object
@@ -114,6 +114,11 @@ def initiate_scan(
scan.tasks = engine.tasks
scan.results_dir = f'{results_dir}/{domain.name}_{scan.id}'
add_gf_patterns = gf_patterns and 'fetch_url' in engine.tasks
+ # add configs to scan object, cfg_ prefix is used to avoid conflicts with other scan object fields
+ scan.cfg_starting_point_path = starting_point_path
+ scan.cfg_excluded_paths = excluded_paths
+ scan.cfg_out_of_scope_subdomains = out_of_scope_subdomains
+
if add_gf_patterns:
scan.used_gf_patterns = ','.join(gf_patterns)
scan.save()
@@ -127,7 +132,7 @@ def initiate_scan(
'engine_id': engine_id,
'domain_id': domain.id,
'results_dir': scan.results_dir,
- 'starting_point_url': starting_point_url,
+ 'starting_point_path': starting_point_path,
'excluded_paths': excluded_paths,
'yaml_configuration': config,
'out_of_scope_subdomains': out_of_scope_subdomains
@@ -152,7 +157,7 @@ def initiate_scan(
# If enable_http_crawl is set, create an initial root HTTP endpoint so that
# HTTP crawling can start somewhere
- http_url = f'{domain.name}{starting_point_url}' if starting_point_url else domain.name
+ http_url = f'{domain.name}{starting_point_path}' if starting_point_path else domain.name
endpoint, _ = save_endpoint(
http_url,
ctx=ctx,
@@ -228,7 +233,7 @@ def initiate_subscan(
engine_id=None,
scan_type=None,
results_dir=RENGINE_RESULTS,
- starting_point_url='',
+ starting_point_path='',
excluded_paths=[],
):
"""Initiate a new subscan.
@@ -239,7 +244,7 @@ def initiate_subscan(
engine_id (int): Engine ID.
scan_type (int): Scan type (periodic, live).
results_dir (str): Results directory.
- starting_point_url (str): URL path. Default: ''
+ starting_point_path (str): URL path. Default: ''
excluded_paths (list): Excluded paths. Default: [], url paths to exclude from scan.
"""
@@ -298,13 +303,13 @@ def initiate_subscan(
'subdomain_id': subdomain.id,
'yaml_configuration': config,
'results_dir': results_dir,
- 'starting_point_url': starting_point_url,
+ 'starting_point_path': starting_point_path,
'excluded_paths': excluded_paths,
}
# Create initial endpoints in DB: find domain HTTP endpoint so that HTTP
# crawling can start somewhere
- base_url = f'{subdomain.name}{starting_point_url}' if starting_point_url else subdomain.name
+ base_url = f'{subdomain.name}{starting_point_path}' if starting_point_path else subdomain.name
endpoint, _ = save_endpoint(
base_url,
crawl=enable_http_crawl,
@@ -406,8 +411,8 @@ def subdomain_discovery(
if not host:
host = self.subdomain.name if self.subdomain else self.domain.name
- if self.starting_point_url:
- logger.warning(f'Ignoring subdomains scan as an URL path filter was passed ({self.starting_point_url}).')
+ if self.starting_point_path:
+ logger.warning(f'Ignoring subdomains scan as an URL path filter was passed ({self.starting_point_path}).')
return
# Config
@@ -1929,7 +1934,7 @@ def fetch_url(self, urls=[], ctx={}, description=None):
if base_url and urlpath:
subdomain = urlparse(base_url)
- url = f'{subdomain.scheme}://{subdomain.netloc}{self.starting_point_url}'
+ url = f'{subdomain.scheme}://{subdomain.netloc}{self.starting_point_path}'
if not validators.url(url):
logger.warning(f'Invalid URL "{url}". Skipping.')
@@ -1938,8 +1943,8 @@ def fetch_url(self, urls=[], ctx={}, description=None):
all_urls.append(url)
# Filter out URLs if a path filter was passed
- if self.starting_point_url:
- all_urls = [url for url in all_urls if self.starting_point_url in url]
+ if self.starting_point_path:
+ all_urls = [url for url in all_urls if self.starting_point_path in url]
# if exclude_paths is found, then remove urls matching those paths
if self.excluded_paths:
@@ -2839,8 +2844,8 @@ def http_crawl(
input_path = f'{self.results_dir}/httpx_input.txt'
history_file = f'{self.results_dir}/commands.txt'
if urls: # direct passing URLs to check
- if self.starting_point_url:
- urls = [u for u in urls if self.starting_point_url in u]
+ if self.starting_point_path:
+ urls = [u for u in urls if self.starting_point_path in u]
with open(input_path, 'w') as f:
f.write('\n'.join(urls))
diff --git a/web/startScan/migrations/0002_auto_20240821_1518.py b/web/startScan/migrations/0002_auto_20240821_1518.py
new file mode 100644
index 00000000..03dd158b
--- /dev/null
+++ b/web/startScan/migrations/0002_auto_20240821_1518.py
@@ -0,0 +1,29 @@
+# Generated by Django 3.2.23 on 2024-08-21 15:18
+
+import django.contrib.postgres.fields
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('startScan', '0001_initial'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='scanhistory',
+ name='cfg_excluded_paths',
+ field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=200), blank=True, default=list, null=True, size=None),
+ ),
+ migrations.AddField(
+ model_name='scanhistory',
+ name='cfg_out_of_scope_subdomains',
+ field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=200), blank=True, default=list, null=True, size=None),
+ ),
+ migrations.AddField(
+ model_name='scanhistory',
+ name='cfg_starting_point_url',
+ field=models.CharField(blank=True, max_length=200, null=True),
+ ),
+ ]
diff --git a/web/startScan/migrations/0003_rename_cfg_starting_point_url_scanhistory_cfg_starting_point_path.py b/web/startScan/migrations/0003_rename_cfg_starting_point_url_scanhistory_cfg_starting_point_path.py
new file mode 100644
index 00000000..e5f005bf
--- /dev/null
+++ b/web/startScan/migrations/0003_rename_cfg_starting_point_url_scanhistory_cfg_starting_point_path.py
@@ -0,0 +1,18 @@
+# Generated by Django 3.2.23 on 2024-08-21 15:40
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('startScan', '0002_auto_20240821_1518'),
+ ]
+
+ operations = [
+ migrations.RenameField(
+ model_name='scanhistory',
+ old_name='cfg_starting_point_url',
+ new_name='cfg_starting_point_path',
+ ),
+ ]
diff --git a/web/startScan/models.py b/web/startScan/models.py
index 0d44638a..b93a0645 100644
--- a/web/startScan/models.py
+++ b/web/startScan/models.py
@@ -48,6 +48,20 @@ class ScanHistory(models.Model):
dorks = models.ManyToManyField('Dork', related_name='dorks', blank=True)
initiated_by = models.ForeignKey(User, on_delete=models.CASCADE, related_name='initiated_scans', blank=True, null=True)
aborted_by = models.ForeignKey(User, on_delete=models.SET_NULL, null=True, blank=True, related_name='aborted_scans')
+ # scan related configs, append config fields with cfg_
+ cfg_out_of_scope_subdomains = ArrayField(
+ models.CharField(max_length=200),
+ blank=True,
+ null=True,
+ default=list
+ )
+ cfg_starting_point_path = models.CharField(max_length=200, blank=True, null=True)
+ cfg_excluded_paths = ArrayField(
+ models.CharField(max_length=200),
+ blank=True,
+ null=True,
+ default=list
+ )
def __str__(self):
diff --git a/web/startScan/templates/startScan/_items/schedule_scan_wizard.html b/web/startScan/templates/startScan/_items/schedule_scan_wizard.html
index d2dbf881..b4e43e0f 100644
--- a/web/startScan/templates/startScan/_items/schedule_scan_wizard.html
+++ b/web/startScan/templates/startScan/_items/schedule_scan_wizard.html
@@ -88,8 +88,8 @@
URL Scope and Exclusions
-
Starting Point URL (Optional)
-
+
Starting Point Path (Optional)
+
Defines where the scan should begin. Leave blank to scan from the root (/) and include all subdomains.
diff --git a/web/startScan/templates/startScan/_items/start_scan_wizard.html b/web/startScan/templates/startScan/_items/start_scan_wizard.html
index 1f2af3ed..1fb0934d 100644
--- a/web/startScan/templates/startScan/_items/start_scan_wizard.html
+++ b/web/startScan/templates/startScan/_items/start_scan_wizard.html
@@ -51,8 +51,8 @@
URL Scope and Exclusions
-
Starting Point URL (Optional)
-
+
Starting Point Path (Optional)
+
Defines where the scan should begin. Leave blank to scan from the root (/) and include all subdomains.
diff --git a/web/startScan/views.py b/web/startScan/views.py
index 94b9ef21..d6908f9c 100644
--- a/web/startScan/views.py
+++ b/web/startScan/views.py
@@ -259,7 +259,7 @@ def start_scan_ui(request, slug, domain_id):
subdomains_in = [s.rstrip() for s in subdomains_in if s]
subdomains_out = request.POST['outOfScopeSubdomainTextarea'].split()
subdomains_out = [s.rstrip() for s in subdomains_out if s]
- starting_point_url = request.POST['startingPointUrl'].strip()
+ starting_point_path = request.POST['startingPointPath'].strip()
excluded_paths = request.POST['excludedPaths'] # string separated by ,
# split excluded paths by ,
excluded_paths = [path.strip() for path in excluded_paths.split(',')]
@@ -284,7 +284,7 @@ def start_scan_ui(request, slug, domain_id):
'results_dir': '/usr/src/scan_results',
'imported_subdomains': subdomains_in,
'out_of_scope_subdomains': subdomains_out,
- 'starting_point_url': starting_point_url,
+ 'starting_point_path': starting_point_path,
'excluded_paths': excluded_paths,
'initiated_by_id': request.user.id
}
@@ -329,7 +329,7 @@ def start_multiple_scan(request, slug):
subdomains_in = [s.rstrip() for s in subdomains_in if s]
subdomains_out = request.POST['outOfScopeSubdomainTextarea'].split()
subdomains_out = [s.rstrip() for s in subdomains_out if s]
- starting_point_url = request.POST['startingPointUrl'].strip()
+ starting_point_path = request.POST['startingPointPath'].strip()
excluded_paths = request.POST['excludedPaths'] # string separated by ,
# split excluded paths by ,
excluded_paths = [path.strip() for path in excluded_paths.split(',')]
@@ -354,7 +354,7 @@ def start_multiple_scan(request, slug):
'initiated_by_id': request.user.id,
'imported_subdomains': subdomains_in,
'out_of_scope_subdomains': subdomains_out,
- 'starting_point_url': starting_point_url,
+ 'starting_point_path': starting_point_path,
'excluded_paths': excluded_paths,
}
@@ -562,7 +562,7 @@ def schedule_scan(request, host_id, slug):
subdomains_in = [s.rstrip() for s in subdomains_in if s]
subdomains_out = request.POST['outOfScopeSubdomainTextarea'].split()
subdomains_out = [s.rstrip() for s in subdomains_out if s]
- starting_point_url = request.POST['startingPointUrl'].strip()
+ starting_point_path = request.POST['startingPointPath'].strip()
excluded_paths = request.POST['excludedPaths'] # string separated by ,
# split excluded paths by ,
excluded_paths = [path.strip() for path in excluded_paths.split(',')]
@@ -596,7 +596,7 @@ def schedule_scan(request, host_id, slug):
'scan_type': SCHEDULED_SCAN,
'imported_subdomains': subdomains_in,
'out_of_scope_subdomains': subdomains_out,
- 'starting_point_url': starting_point_url,
+ 'starting_point_path': starting_point_path,
'excluded_paths': excluded_paths,
'initiated_by_id': request.user.id
}
@@ -617,7 +617,7 @@ def schedule_scan(request, host_id, slug):
'scan_type': SCHEDULED_SCAN,
'imported_subdomains': subdomains_in,
'out_of_scope_subdomains': subdomains_out,
- 'starting_point_url': starting_point_url,
+ 'starting_point_path': starting_point_path,
'excluded_paths': excluded_paths,
'initiated_by_id': request.user.id
}
@@ -745,7 +745,7 @@ def start_organization_scan(request, id, slug):
subdomains_in = [s.rstrip() for s in subdomains_in if s]
subdomains_out = request.POST['outOfScopeSubdomainTextarea'].split()
subdomains_out = [s.rstrip() for s in subdomains_out if s]
- starting_point_url = request.POST['startingPointUrl'].strip()
+ starting_point_path = request.POST['startingPointPath'].strip()
excluded_paths = request.POST['excludedPaths'] # string separated by ,
# split excluded paths by ,
excluded_paths = [path.strip() for path in excluded_paths.split(',')]
@@ -768,7 +768,7 @@ def start_organization_scan(request, id, slug):
'initiated_by_id': request.user.id,
'imported_subdomains': subdomains_in,
'out_of_scope_subdomains': subdomains_out,
- 'starting_point_url': starting_point_url,
+ 'starting_point_path': starting_point_path,
'excluded_paths': excluded_paths,
}
initiate_scan.apply_async(kwargs=kwargs)
@@ -814,7 +814,7 @@ def schedule_organization_scan(request, slug, id):
subdomains_in = [s.rstrip() for s in subdomains_in if s]
subdomains_out = request.POST['outOfScopeSubdomainTextarea'].split()
subdomains_out = [s.rstrip() for s in subdomains_out if s]
- starting_point_url = request.POST['startingPointUrl'].strip()
+ starting_point_path = request.POST['startingPointPath'].strip()
excluded_paths = request.POST['excludedPaths'] # string separated by ,
# split excluded paths by ,
excluded_paths = [path.strip() for path in excluded_paths.split(',')]
@@ -852,7 +852,7 @@ def schedule_organization_scan(request, slug, id):
'initiated_by_id': request.user.id,
'imported_subdomains': subdomains_in,
'out_of_scope_subdomains': subdomains_out,
- 'starting_point_url': starting_point_url,
+ 'starting_point_path': starting_point_path,
'excluded_paths': excluded_paths,
})
PeriodicTask.objects.create(
@@ -876,7 +876,7 @@ def schedule_organization_scan(request, slug, id):
'initiated_by_id': request.user.id,
'imported_subdomains': subdomains_in,
'out_of_scope_subdomains': subdomains_out,
- 'starting_point_url': starting_point_url,
+ 'starting_point_path': starting_point_path,
'excluded_paths': excluded_paths,
})
PeriodicTask.objects.create(clocked=clock,