WEBDIENSTE/rengine
1
0
Fork 0
mirror of https://github.com/yogeshojha/rengine.git synced 2026-09-27 20:24:54 +02:00
Code Issues Packages Projects Releases Wiki Activity

refactor starting point url to starting point path

Browse Source
This commit is contained in:
Yogesh Ojha
2024-08-21 21:11:08 +05:30
parent cdd216ee8d
commit 2f539a70d1
8 changed files with 99 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
web/reNgine/celery_custom_task.py
+1 -1
View File
@@ -67,7 +67,7 @@ class RengineTask(Task):
self.subscan_id = ctx.get('subscan_id')
self.engine_id = ctx.get('engine_id')
self.filename = ctx.get('filename')
self.starting_point_url = ctx.get('starting_point_url', '')
self.starting_point_path = ctx.get('starting_point_path', '')
self.excluded_paths = ctx.get('excluded_paths', [])
self.results_dir = ctx.get('results_dir', RENGINE_RESULTS)
self.yaml_configuration = ctx.get('yaml_configuration', {})
web/reNgine/tasks.py
+21 -16
View File
@@ -57,7 +57,7 @@ def initiate_scan(
imported_subdomains=[],
out_of_scope_subdomains=[],
initiated_by_id=None,
starting_point_url='',
starting_point_path='',
excluded_paths=[],
):
"""Initiate a new scan.
@@ -70,7 +70,7 @@ def initiate_scan(
results_dir (str): Results directory.
imported_subdomains (list): Imported subdomains.
out_of_scope_subdomains (list): Out-of-scope subdomains.
starting_point_url (str): URL path. Default: '' Defined where to start the scan.
starting_point_path (str): URL path. Default: '' Defined where to start the scan.
initiated_by (int): User ID initiating the scan.
excluded_paths (list): Excluded paths. Default: [], url paths to exclude from scan.
"""
@@ -92,7 +92,7 @@ def initiate_scan(
domain.save()
# Get path filter
starting_point_url = starting_point_url.rstrip('/')
starting_point_path = starting_point_path.rstrip('/')
# for live scan scan history id is passed as scan_history_id
# and no need to create scan_history object
@@ -114,6 +114,11 @@ def initiate_scan(
scan.tasks = engine.tasks
scan.results_dir = f'{results_dir}/{domain.name}_{scan.id}'
add_gf_patterns = gf_patterns and 'fetch_url' in engine.tasks
# add configs to scan object, cfg_ prefix is used to avoid conflicts with other scan object fields
scan.cfg_starting_point_path = starting_point_path
scan.cfg_excluded_paths = excluded_paths
scan.cfg_out_of_scope_subdomains = out_of_scope_subdomains
if add_gf_patterns:
scan.used_gf_patterns = ','.join(gf_patterns)
scan.save()
@@ -127,7 +132,7 @@ def initiate_scan(
'engine_id': engine_id,
'domain_id': domain.id,
'results_dir': scan.results_dir,
'starting_point_url': starting_point_url,
'starting_point_path': starting_point_path,
'excluded_paths': excluded_paths,
'yaml_configuration': config,
'out_of_scope_subdomains': out_of_scope_subdomains
@@ -152,7 +157,7 @@ def initiate_scan(
# If enable_http_crawl is set, create an initial root HTTP endpoint so that
# HTTP crawling can start somewhere
http_url = f'{domain.name}{starting_point_url}' if starting_point_url else domain.name
http_url = f'{domain.name}{starting_point_path}' if starting_point_path else domain.name
endpoint, _ = save_endpoint(
http_url,
ctx=ctx,
@@ -228,7 +233,7 @@ def initiate_subscan(
engine_id=None,
scan_type=None,
results_dir=RENGINE_RESULTS,
starting_point_url='',
starting_point_path='',
excluded_paths=[],
):
"""Initiate a new subscan.
@@ -239,7 +244,7 @@ def initiate_subscan(
engine_id (int): Engine ID.
scan_type (int): Scan type (periodic, live).
results_dir (str): Results directory.
starting_point_url (str): URL path. Default: ''
starting_point_path (str): URL path. Default: ''
excluded_paths (list): Excluded paths. Default: [], url paths to exclude from scan.
"""
@@ -298,13 +303,13 @@ def initiate_subscan(
'subdomain_id': subdomain.id,
'yaml_configuration': config,
'results_dir': results_dir,
'starting_point_url': starting_point_url,
'starting_point_path': starting_point_path,
'excluded_paths': excluded_paths,
}
# Create initial endpoints in DB: find domain HTTP endpoint so that HTTP
# crawling can start somewhere
base_url = f'{subdomain.name}{starting_point_url}' if starting_point_url else subdomain.name
base_url = f'{subdomain.name}{starting_point_path}' if starting_point_path else subdomain.name
endpoint, _ = save_endpoint(
base_url,
crawl=enable_http_crawl,
@@ -406,8 +411,8 @@ def subdomain_discovery(
if not host:
host = self.subdomain.name if self.subdomain else self.domain.name
if self.starting_point_url:
logger.warning(f'Ignoring subdomains scan as an URL path filter was passed ({self.starting_point_url}).')
if self.starting_point_path:
logger.warning(f'Ignoring subdomains scan as an URL path filter was passed ({self.starting_point_path}).')
return
# Config
@@ -1929,7 +1934,7 @@ def fetch_url(self, urls=[], ctx={}, description=None):
if base_url and urlpath:
subdomain = urlparse(base_url)
url = f'{subdomain.scheme}://{subdomain.netloc}{self.starting_point_url}'
url = f'{subdomain.scheme}://{subdomain.netloc}{self.starting_point_path}'
if not validators.url(url):
logger.warning(f'Invalid URL "{url}". Skipping.')
@@ -1938,8 +1943,8 @@ def fetch_url(self, urls=[], ctx={}, description=None):
all_urls.append(url)
# Filter out URLs if a path filter was passed
if self.starting_point_url:
all_urls = [url for url in all_urls if self.starting_point_url in url]
if self.starting_point_path:
all_urls = [url for url in all_urls if self.starting_point_path in url]
# if exclude_paths is found, then remove urls matching those paths
if self.excluded_paths:
@@ -2839,8 +2844,8 @@ def http_crawl(
input_path = f'{self.results_dir}/httpx_input.txt'
history_file = f'{self.results_dir}/commands.txt'
if urls: # direct passing URLs to check
if self.starting_point_url:
urls = [u for u in urls if self.starting_point_url in u]
if self.starting_point_path:
urls = [u for u in urls if self.starting_point_path in u]
with open(input_path, 'w') as f:
f.write('\n'.join(urls))
web/startScan/migrations/0002_auto_20240821_1518.py
+29
View File
@@ -0,0 +1,29 @@
# Generated by Django 3.2.23 on 2024-08-21 15:18
import django.contrib.postgres.fields
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('startScan', '0001_initial'),
]
operations = [
migrations.AddField(
model_name='scanhistory',
name='cfg_excluded_paths',
field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=200), blank=True, default=list, null=True, size=None),
),
migrations.AddField(
model_name='scanhistory',
name='cfg_out_of_scope_subdomains',
field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=200), blank=True, default=list, null=True, size=None),
),
migrations.AddField(
model_name='scanhistory',
name='cfg_starting_point_url',
field=models.CharField(blank=True, max_length=200, null=True),
),
]
web/startScan/migrations/0003_rename_cfg_starting_point_url_scanhistory_cfg_starting_point_path.py
+18
View File
@@ -0,0 +1,18 @@
# Generated by Django 3.2.23 on 2024-08-21 15:40
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('startScan', '0002_auto_20240821_1518'),
]
operations = [
migrations.RenameField(
model_name='scanhistory',
old_name='cfg_starting_point_url',
new_name='cfg_starting_point_path',
),
]
web/startScan/models.py
+14
View File
@@ -48,6 +48,20 @@ class ScanHistory(models.Model):
dorks = models.ManyToManyField('Dork', related_name='dorks', blank=True)
initiated_by = models.ForeignKey(User, on_delete=models.CASCADE, related_name='initiated_scans', blank=True, null=True)
aborted_by = models.ForeignKey(User, on_delete=models.SET_NULL, null=True, blank=True, related_name='aborted_scans')
# scan related configs, append config fields with cfg_
cfg_out_of_scope_subdomains = ArrayField(
models.CharField(max_length=200),
blank=True,
null=True,
default=list
)
cfg_starting_point_path = models.CharField(max_length=200, blank=True, null=True)
cfg_excluded_paths = ArrayField(
models.CharField(max_length=200),
blank=True,
null=True,
default=list
)
def __str__(self):
web/startScan/templates/startScan/_items/schedule_scan_wizard.html
+2 -2
View File
@@ -88,8 +88,8 @@
<h3>URL Scope and Exclusions</h3>
<div class="mb-4">
<div class="mb-3">
<h4 class="text-info">Starting Point URL (Optional)</h4>
<input type="email" class="form-control" id="startingPointUrl" placeholder="e.g. /home" name="startingPointUrl">
<h4 class="text-info">Starting Point Path (Optional)</h4>
<input type="email" class="form-control" id="startingPointPath" placeholder="e.g. /home" name="startingPointPath">
<small class="form-text text-muted">
Defines where the scan should begin. Leave blank to scan from the root (/) and include all subdomains.
<br>
web/startScan/templates/startScan/_items/start_scan_wizard.html
+2 -2
View File
@@ -51,8 +51,8 @@
<h4>URL Scope and Exclusions</h4>
<div class="mb-4">
<div class="mb-3">
<h4 class="text-info">Starting Point URL (Optional)</h4>
<input type="email" class="form-control" id="startingPointUrl" placeholder="e.g. /home" name="startingPointUrl">
<h4 class="text-info">Starting Point Path (Optional)</h4>
<input type="email" class="form-control" id="startingPointPath" placeholder="e.g. /home" name="startingPointPath">
<small class="form-text text-muted">
Defines where the scan should begin. Leave blank to scan from the root (/) and include all subdomains.
</br>
web/startScan/views.py
+12 -12
View File
@@ -259,7 +259,7 @@ def start_scan_ui(request, slug, domain_id):
subdomains_in = [s.rstrip() for s in subdomains_in if s]
subdomains_out = request.POST['outOfScopeSubdomainTextarea'].split()
subdomains_out = [s.rstrip() for s in subdomains_out if s]
starting_point_url = request.POST['startingPointUrl'].strip()
starting_point_path = request.POST['startingPointPath'].strip()
excluded_paths = request.POST['excludedPaths'] # string separated by ,
# split excluded paths by ,
excluded_paths = [path.strip() for path in excluded_paths.split(',')]
@@ -284,7 +284,7 @@ def start_scan_ui(request, slug, domain_id):
'results_dir': '/usr/src/scan_results',
'imported_subdomains': subdomains_in,
'out_of_scope_subdomains': subdomains_out,
'starting_point_url': starting_point_url,
'starting_point_path': starting_point_path,
'excluded_paths': excluded_paths,
'initiated_by_id': request.user.id
}
@@ -329,7 +329,7 @@ def start_multiple_scan(request, slug):
subdomains_in = [s.rstrip() for s in subdomains_in if s]
subdomains_out = request.POST['outOfScopeSubdomainTextarea'].split()
subdomains_out = [s.rstrip() for s in subdomains_out if s]
starting_point_url = request.POST['startingPointUrl'].strip()
starting_point_path = request.POST['startingPointPath'].strip()
excluded_paths = request.POST['excludedPaths'] # string separated by ,
# split excluded paths by ,
excluded_paths = [path.strip() for path in excluded_paths.split(',')]
@@ -354,7 +354,7 @@ def start_multiple_scan(request, slug):
'initiated_by_id': request.user.id,
'imported_subdomains': subdomains_in,
'out_of_scope_subdomains': subdomains_out,
'starting_point_url': starting_point_url,
'starting_point_path': starting_point_path,
'excluded_paths': excluded_paths,
}
@@ -562,7 +562,7 @@ def schedule_scan(request, host_id, slug):
subdomains_in = [s.rstrip() for s in subdomains_in if s]
subdomains_out = request.POST['outOfScopeSubdomainTextarea'].split()
subdomains_out = [s.rstrip() for s in subdomains_out if s]
starting_point_url = request.POST['startingPointUrl'].strip()
starting_point_path = request.POST['startingPointPath'].strip()
excluded_paths = request.POST['excludedPaths'] # string separated by ,
# split excluded paths by ,
excluded_paths = [path.strip() for path in excluded_paths.split(',')]
@@ -596,7 +596,7 @@ def schedule_scan(request, host_id, slug):
'scan_type': SCHEDULED_SCAN,
'imported_subdomains': subdomains_in,
'out_of_scope_subdomains': subdomains_out,
'starting_point_url': starting_point_url,
'starting_point_path': starting_point_path,
'excluded_paths': excluded_paths,
'initiated_by_id': request.user.id
}
@@ -617,7 +617,7 @@ def schedule_scan(request, host_id, slug):
'scan_type': SCHEDULED_SCAN,
'imported_subdomains': subdomains_in,
'out_of_scope_subdomains': subdomains_out,
'starting_point_url': starting_point_url,
'starting_point_path': starting_point_path,
'excluded_paths': excluded_paths,
'initiated_by_id': request.user.id
}
@@ -745,7 +745,7 @@ def start_organization_scan(request, id, slug):
subdomains_in = [s.rstrip() for s in subdomains_in if s]
subdomains_out = request.POST['outOfScopeSubdomainTextarea'].split()
subdomains_out = [s.rstrip() for s in subdomains_out if s]
starting_point_url = request.POST['startingPointUrl'].strip()
starting_point_path = request.POST['startingPointPath'].strip()
excluded_paths = request.POST['excludedPaths'] # string separated by ,
# split excluded paths by ,
excluded_paths = [path.strip() for path in excluded_paths.split(',')]
@@ -768,7 +768,7 @@ def start_organization_scan(request, id, slug):
'initiated_by_id': request.user.id,
'imported_subdomains': subdomains_in,
'out_of_scope_subdomains': subdomains_out,
'starting_point_url': starting_point_url,
'starting_point_path': starting_point_path,
'excluded_paths': excluded_paths,
}
initiate_scan.apply_async(kwargs=kwargs)
@@ -814,7 +814,7 @@ def schedule_organization_scan(request, slug, id):
subdomains_in = [s.rstrip() for s in subdomains_in if s]
subdomains_out = request.POST['outOfScopeSubdomainTextarea'].split()
subdomains_out = [s.rstrip() for s in subdomains_out if s]
starting_point_url = request.POST['startingPointUrl'].strip()
starting_point_path = request.POST['startingPointPath'].strip()
excluded_paths = request.POST['excludedPaths'] # string separated by ,
# split excluded paths by ,
excluded_paths = [path.strip() for path in excluded_paths.split(',')]
@@ -852,7 +852,7 @@ def schedule_organization_scan(request, slug, id):
'initiated_by_id': request.user.id,
'imported_subdomains': subdomains_in,
'out_of_scope_subdomains': subdomains_out,
'starting_point_url': starting_point_url,
'starting_point_path': starting_point_path,
'excluded_paths': excluded_paths,
})
PeriodicTask.objects.create(
@@ -876,7 +876,7 @@ def schedule_organization_scan(request, slug, id):
'initiated_by_id': request.user.id,
'imported_subdomains': subdomains_in,
'out_of_scope_subdomains': subdomains_out,
'starting_point_url': starting_point_url,
'starting_point_path': starting_point_path,
'excluded_paths': excluded_paths,
})
PeriodicTask.objects.create(clocked=clock,