mirror of
https://github.com/j3ssie/osmedeus.git
synced 2026-08-25 09:02:29 +02:00
j3ssie
f5840272c5
Major features: - Add run registry for tracking active runs with PID management - Add API-based run cancellation with process termination - Add event trigger input vars syntax for multi-variable extraction - Add filter_functions with utility function support in triggers - Add event envelope injection for full event context in workflows - Add write coordinator for batched database operations API improvements: - Add logout endpoint and diffs endpoints for assets/vulnerabilities - Add step-results listing endpoint - Update schedule model with target, workspace, params fields - Change run_id to run_uuid across API responses Performance: - Add compiled JS program caching for 60-80% faster loop conditions - Add parallel shard rendering for 20-40% faster workflow startup - Add memory-mapped I/O for large file line counting - Add efficient output buffer combining in runners - Add mtime-based cache invalidation for workflow loader Other changes: - Rename trigger field from trigger to triggers in workflow YAML - Disable pongo2 HTML autoescape for shell command templates - Update JWT expiration default to 1440 minutes (1 day) - Change CORS default to reflect-origin for credentials support - Add source_type field to events (run, eval, api) - Skip copying core Unix tools to external-binaries
264 lines
11 KiB
YAML
264 lines
11 KiB
YAML
name: web-reconnaissance
|
|
kind: module
|
|
desc: Comprehensive web reconnaissance module demonstrating advanced workflow features
|
|
|
|
params:
|
|
- name: target
|
|
required: true
|
|
- name: output_dir
|
|
default: /tmp/osm-web-recon
|
|
- name: threads
|
|
default: "5"
|
|
- name: subfinderThreads
|
|
default: "{{threads * 4}}"
|
|
- name: httpxThreads
|
|
default: "{{threads * 2}}"
|
|
- name: nucleiThreads
|
|
default: "{{threads}}"
|
|
- name: screenshotThreads
|
|
default: "5"
|
|
- name: httpxTimeout
|
|
default: "10"
|
|
- name: nucleiTimeout
|
|
default: "3600"
|
|
- name: enableScreenshots
|
|
default: "true"
|
|
- name: enableNuclei
|
|
default: "true"
|
|
- name: nucleiSeverity
|
|
default: "critical,high,medium"
|
|
- name: subfinderConfig
|
|
default: "{{Data}}/external-configs/subfinder-provider.yaml"
|
|
|
|
steps:
|
|
# ============================================================
|
|
# Phase 1: Validate Dependencies
|
|
# ============================================================
|
|
- name: validate-dependencies
|
|
type: function
|
|
function: |
|
|
file_exists("{{Binaries}}/subfinder") &&
|
|
file_exists("{{Binaries}}/assetfinder") &&
|
|
file_exists("{{Binaries}}/httpx")
|
|
exports:
|
|
deps_valid: "output"
|
|
on_error:
|
|
- action: log
|
|
message: "Required binaries not found"
|
|
- action: abort
|
|
|
|
# ============================================================
|
|
# Phase 2: Parallel Subdomain Enumeration
|
|
# ============================================================
|
|
- name: subdomain-enumeration
|
|
type: parallel-steps
|
|
parallel_steps:
|
|
- name: run-subfinder
|
|
type: bash
|
|
command: "{{Binaries}}/subfinder -d {{Target}} -provider-config {{subfinderConfig}} -t {{subfinderThreads}} -o {{Output}}/web-recon/subdomains/{{TargetSpace}}-subfinder.txt -silent"
|
|
timeout: 600
|
|
on_error:
|
|
- action: log
|
|
message: "Subfinder failed, continuing with other tools"
|
|
- action: continue
|
|
|
|
- name: run-assetfinder
|
|
type: bash
|
|
command: "{{Binaries}}/assetfinder -subs-only {{Target}} > {{Output}}/web-recon/subdomains/{{TargetSpace}}-assetfinder.txt"
|
|
timeout: 300
|
|
on_error:
|
|
- action: continue
|
|
|
|
- name: run-findomain
|
|
type: bash
|
|
command: "{{Binaries}}/findomain -u {{Output}}/web-recon/subdomains/{{TargetSpace}}-findomain.txt -t {{Target}} 2>/dev/null"
|
|
timeout: 300
|
|
on_error:
|
|
- action: continue
|
|
|
|
# ============================================================
|
|
# Phase 3: Merge and Deduplicate Subdomains
|
|
# ============================================================
|
|
- name: merge-subdomains
|
|
type: bash
|
|
commands:
|
|
- "cat {{Output}}/web-recon/subdomains/{{TargetSpace}}-*.txt 2>/dev/null | sort -u > {{Output}}/web-recon/subdomains/all-{{TargetSpace}}.txt"
|
|
- "cat {{Output}}/web-recon/subdomains/all-{{TargetSpace}}.txt | {{Binaries}}/cleansub -t '{{Target}}' > {{Output}}/web-recon/subdomains/final-{{TargetSpace}}.txt 2>/dev/null || cp {{Output}}/web-recon/subdomains/all-{{TargetSpace}}.txt {{Output}}/web-recon/subdomains/final-{{TargetSpace}}.txt"
|
|
exports:
|
|
subdomains_file: "{{Output}}/web-recon/subdomains/final-{{TargetSpace}}.txt"
|
|
|
|
- name: count-subdomains
|
|
type: function
|
|
function: |
|
|
var count = file_length("{{subdomains_file}}");
|
|
return count > 0 ? "true" : "false";
|
|
exports:
|
|
subdomain_count: "{{Result}}"
|
|
has_subdomains: "{{Result}}"
|
|
|
|
# Decision: Skip remaining steps if no subdomains found
|
|
- name: check-subdomain-results
|
|
type: bash
|
|
command: "echo {{subdomain_count}}"
|
|
decision:
|
|
switch: "{{has_subdomains}}"
|
|
cases:
|
|
"false":
|
|
goto: generate-empty-report
|
|
default:
|
|
goto: http-probing
|
|
|
|
# ============================================================
|
|
# Phase 4: HTTP Probing
|
|
# ============================================================
|
|
- name: http-probing
|
|
type: bash
|
|
command: "{{Binaries}}/httpx -l {{subdomains_file}} -threads {{httpxThreads}} -timeout {{httpxTimeout}} -silent -o {{Output}}/web-recon/probing/live-{{TargetSpace}}.txt -json -output {{Output}}/web-recon/probing/httpx-{{TargetSpace}}.json"
|
|
timeout: 900
|
|
exports:
|
|
live_hosts_file: "{{Output}}/web-recon/probing/live-{{TargetSpace}}.txt"
|
|
on_error:
|
|
- action: log
|
|
message: "HTTP probing failed"
|
|
- action: run
|
|
step: fallback-probing
|
|
|
|
- name: fallback-probing
|
|
type: bash
|
|
pre_condition: "!file_exists('{{Output}}/web-recon/probing/live-{{TargetSpace}}.txt')"
|
|
command: "cat {{subdomains_file}} | xargs -I {} curl -s -o /dev/null -w '%{http_code} {}\\n' http://{} 2>/dev/null | grep '^200' | awk '{print $2}' > {{Output}}/web-recon/probing/live-{{TargetSpace}}.txt"
|
|
exports:
|
|
live_hosts_file: "{{Output}}/web-recon/probing/live-{{TargetSpace}}.txt"
|
|
|
|
- name: count-live-hosts
|
|
type: function
|
|
function: file_length("{{live_hosts_file}}")
|
|
exports:
|
|
live_host_count: "output"
|
|
|
|
# ============================================================
|
|
# Phase 5: Parallel Analysis (Screenshots + Nuclei)
|
|
# ============================================================
|
|
- name: parallel-analysis
|
|
type: parallel-steps
|
|
parallel_steps:
|
|
# Screenshot capture using Docker
|
|
- name: capture-screenshots
|
|
type: remote-bash
|
|
pre_condition: '"{{enableScreenshots}}" == "true" && parse_int("{{live_host_count}}") > 0'
|
|
step_runner: docker
|
|
step_runner_config:
|
|
image: projectdiscovery/katana:latest
|
|
volumes:
|
|
- "{{Output}}/web-recon:/output"
|
|
workdir: /output
|
|
env:
|
|
TARGETS_FILE: "/output/probing/live-{{TargetSpace}}.txt"
|
|
command: |
|
|
echo "Capturing screenshots for live hosts..."
|
|
cat $TARGETS_FILE | head -20
|
|
timeout: 1800
|
|
on_error:
|
|
- action: log
|
|
message: "Screenshot capture failed"
|
|
- action: continue
|
|
|
|
# Nuclei vulnerability scanning using Docker
|
|
- name: nuclei-scan
|
|
type: remote-bash
|
|
pre_condition: '"{{enableNuclei}}" == "true" && parse_int("{{live_host_count}}") > 0'
|
|
step_runner: docker
|
|
step_runner_config:
|
|
image: projectdiscovery/nuclei:latest
|
|
volumes:
|
|
- "{{Output}}/web-recon:/output"
|
|
workdir: /output
|
|
env:
|
|
SEVERITY: "{{nucleiSeverity}}"
|
|
THREADS: "{{nucleiThreads}}"
|
|
command: |
|
|
nuclei -l /output/probing/live-{{TargetSpace}}.txt \
|
|
-severity $SEVERITY \
|
|
-c $THREADS \
|
|
-json-export /output/nuclei/results-{{TargetSpace}}.json \
|
|
-silent
|
|
timeout: 3600
|
|
exports:
|
|
nuclei_results: "{{Output}}/web-recon/nuclei/results-{{TargetSpace}}.json"
|
|
on_error:
|
|
- action: log
|
|
message: "Nuclei scan failed"
|
|
- action: continue
|
|
|
|
# ============================================================
|
|
# Phase 6: Foreach - Detailed Host Analysis
|
|
# ============================================================
|
|
- name: detailed-host-analysis
|
|
type: foreach
|
|
pre_condition: 'parse_int("{{live_host_count}}") > 0 && parse_int("{{live_host_count}}") < 50'
|
|
input: "{{live_hosts_file}}"
|
|
variable: host
|
|
threads: 5
|
|
step:
|
|
name: analyze-single-host
|
|
type: bash
|
|
command: |
|
|
echo "Analyzing [[host]]..."
|
|
curl -s -I "[[host]]" 2>/dev/null | head -20 >> {{Output}}/web-recon/probing/headers-{{TargetSpace}}.txt
|
|
echo "---" >> {{Output}}/web-recon/probing/headers-{{TargetSpace}}.txt
|
|
timeout: 30
|
|
|
|
# ============================================================
|
|
# Phase 7: Result Processing and Reporting
|
|
# ============================================================
|
|
- name: process-nuclei-results
|
|
type: function
|
|
pre_condition: 'file_exists("{{Output}}/web-recon/nuclei/results-{{TargetSpace}}.json")'
|
|
parallel_functions:
|
|
- db_vuln_critical("{{Output}}/web-recon/nuclei/results-{{TargetSpace}}.json")
|
|
- db_vuln_high("{{Output}}/web-recon/nuclei/results-{{TargetSpace}}.json")
|
|
- db_vuln_medium("{{Output}}/web-recon/nuclei/results-{{TargetSpace}}.json")
|
|
exports:
|
|
vuln_stats: "output"
|
|
|
|
- name: generate-report
|
|
type: bash
|
|
commands:
|
|
- |
|
|
echo "=== Web Reconnaissance Report ===" > {{Output}}/web-recon/final-report-{{TargetSpace}}.txt
|
|
echo "Target: {{Target}}" >> {{Output}}/web-recon/final-report-{{TargetSpace}}.txt
|
|
echo "Workspace: {{TargetSpace}}" >> {{Output}}/web-recon/final-report-{{TargetSpace}}.txt
|
|
echo "Date: $(date)" >> {{Output}}/web-recon/final-report-{{TargetSpace}}.txt
|
|
echo "" >> {{Output}}/web-recon/final-report-{{TargetSpace}}.txt
|
|
echo "=== Statistics ===" >> {{Output}}/web-recon/final-report-{{TargetSpace}}.txt
|
|
echo "Total Subdomains: {{subdomain_count}}" >> {{Output}}/web-recon/final-report-{{TargetSpace}}.txt
|
|
echo "Live Hosts: {{live_host_count}}" >> {{Output}}/web-recon/final-report-{{TargetSpace}}.txt
|
|
echo "" >> {{Output}}/web-recon/final-report-{{TargetSpace}}.txt
|
|
echo "=== Live Hosts ===" >> {{Output}}/web-recon/final-report-{{TargetSpace}}.txt
|
|
cat {{live_hosts_file}} >> {{Output}}/web-recon/final-report-{{TargetSpace}}.txt 2>/dev/null || echo "No live hosts found"
|
|
- "cp {{live_hosts_file}} {{Output}}/web-recon/live-hosts-{{TargetSpace}}.txt 2>/dev/null || touch {{Output}}/web-recon/live-hosts-{{TargetSpace}}.txt"
|
|
- "cp {{Output}}/web-recon/nuclei/results-{{TargetSpace}}.json {{Output}}/web-recon/vulnerabilities-{{TargetSpace}}.json 2>/dev/null || echo '[]' > {{Output}}/web-recon/vulnerabilities-{{TargetSpace}}.json"
|
|
|
|
- name: generate-empty-report
|
|
type: bash
|
|
pre_condition: '"{{has_subdomains}}" == "false"'
|
|
commands:
|
|
- |
|
|
echo "=== Web Reconnaissance Report ===" > {{Output}}/web-recon/final-report-{{TargetSpace}}.txt
|
|
echo "Target: {{Target}}" >> {{Output}}/web-recon/final-report-{{TargetSpace}}.txt
|
|
echo "No subdomains found for target." >> {{Output}}/web-recon/final-report-{{TargetSpace}}.txt
|
|
- "touch {{Output}}/web-recon/live-hosts-{{TargetSpace}}.txt"
|
|
- "echo '[]' > {{Output}}/web-recon/vulnerabilities-{{TargetSpace}}.json"
|
|
|
|
# ============================================================
|
|
# Phase 8: Cleanup and Notifications
|
|
# ============================================================
|
|
- name: final-cleanup
|
|
type: function
|
|
function: SortU("{{Output}}/web-recon/live-hosts-{{TargetSpace}}.txt")
|
|
|
|
- name: notify-completion
|
|
type: function
|
|
pre_condition: 'parse_int("{{subdomain_count}}") > 0'
|
|
function: printf("Scan complete: {{subdomain_count}} subdomains, {{live_host_count}} live hosts")
|