diff --git a/web/reNgine/common_func.py b/web/reNgine/common_func.py index cb0f1ad1..2c7bdbd8 100644 --- a/web/reNgine/common_func.py +++ b/web/reNgine/common_func.py @@ -458,6 +458,23 @@ def sanitize_url(http_url): url = url._replace(netloc=url.netloc.replace(':443', '')) return url.geturl().rstrip('/') +def extract_path_from_url(url): + parsed_url = urlparse(url) + + # Reconstruct the URL without scheme and netloc + reconstructed_url = parsed_url.path + + if reconstructed_url.startswith('/'): + reconstructed_url = reconstructed_url[1:] # Remove the first slash + + if parsed_url.params: + reconstructed_url += ';' + parsed_url.params + if parsed_url.query: + reconstructed_url += '?' + parsed_url.query + if parsed_url.fragment: + reconstructed_url += '#' + parsed_url.fragment + + return reconstructed_url #-------# # Utils # diff --git a/web/reNgine/tasks.py b/web/reNgine/tasks.py index 52d969e9..c074c5f9 100644 --- a/web/reNgine/tasks.py +++ b/web/reNgine/tasks.py @@ -10,6 +10,7 @@ import xmltodict import yaml import tldextract import concurrent.futures +import base64 from datetime import datetime from urllib.parse import urlparse @@ -1662,49 +1663,75 @@ def dir_file_fuzz(self, ctx={}, description=None): history_file=self.history_file, scan_id=self.scan_id, activity_id=self.activity_id): + + # Empty line, continue to the next record if not isinstance(line, dict): continue + + # Append line to results results.append(line) - name = line['input'].get('FUZZ') + + # Retrieve FFUF output + url = line['url'] + # Extract path and convert to base64 (need byte string encode & decode) + name = base64.b64encode(extract_path_from_url(url).encode()).decode() length = line['length'] status = line['status'] words = line['words'] - url = line['url'] lines = line['lines'] content_type = line['content-type'] duration = line['duration'] + + # If name empty log error and continue if not name: logger.error(f'FUZZ not found for "{url}"') continue + + # Get or create endpoint from URL endpoint, created = save_endpoint(url, crawl=False, ctx=ctx) + + # Continue to next line if endpoint returned is None + if endpoint == None: + continue + + # Save endpoint data from FFUF output endpoint.http_status = status endpoint.content_length = length endpoint.response_time = duration / 1000000000 - endpoint.save() - if created: - urls.append(endpoint.http_url) - endpoint.status = status endpoint.content_type = content_type endpoint.content_length = length + endpoint.save() + + # Save directory file output from FFUF output dfile, created = DirectoryFile.objects.get_or_create( name=name, length=length, words=words, lines=lines, content_type=content_type, - url=url) - dfile.http_status = status - dfile.save() - # if created: - # logger.warning(f'Found new directory or file {url}') - dirscan.directory_files.add(dfile) - dirscan.save() + url=url, + http_status=status) + # Log newly created file or directory if debug activated + if created and DEBUG: + logger.warning(f'Found new directory or file {url}') + + # Add file to current dirscan + dirscan.directory_files.add(dfile) + + # Add subscan relation to dirscan if exists if self.subscan: dirscan.dir_subscan_ids.add(self.subscan) - subdomain_name = get_subdomain_from_url(endpoint.http_url) - subdomain = Subdomain.objects.get(name=subdomain_name, scan_history=self.scan) + # Save dirscan datas + dirscan.save() + + # Get subdomain and add dirscan + if ctx['subdomain_id'] > 0: + subdomain = Subdomain.objects.get(id=ctx['subdomain_id']) + else: + subdomain_name = get_subdomain_from_url(endpoint.http_url) + subdomain = Subdomain.objects.get(name=subdomain_name, scan_history=self.scan) subdomain.directories.add(dirscan) subdomain.save()