From e2c742b0171ee773c1747a2b2e5404dc0618a034 Mon Sep 17 00:00:00 2001 From: j3ssie Date: Mon, 9 Sep 2019 11:32:56 +0700 Subject: [PATCH] Fix some bug on gitscan --- lib/alias/git_format | 18 ++++++++++------- lib/alias/gitrecon | 10 ++++++---- lib/alias/portscan | 2 +- lib/core/dbutils.py | 6 ++++-- lib/core/utils.py | 7 ++++++- lib/mode/routine.py | 1 - lib/workflow/direct.py | 10 +++++----- scripts/clean.py | 2 +- scripts/reload.py | 44 ++++++++++++++++++++++++++++++++++++++++++ server/api/models.py | 2 +- 10 files changed, 79 insertions(+), 23 deletions(-) create mode 100755 scripts/reload.py diff --git a/lib/alias/git_format b/lib/alias/git_format index cfdf66a..81f2a52 100755 --- a/lib/alias/git_format +++ b/lib/alias/git_format @@ -9,8 +9,8 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) from core import utils -# git_check -i /tmp/target.txt -o '$WORKSPACE/gitscan/$OUTPUT' -# git_check - repo:sda +# git_format -i /tmp/target.txt -o '$WORKSPACE/gitscan/$OUTPUT' +# git_format -i 'repo:sda' -o '$WORKSPACE/gitscan/$OUTPUT' def parse_input(data, out): repo = f'{out}-repo.txt' @@ -19,13 +19,17 @@ def parse_input(data, out): utils.just_append(repo, data.strip()) else: key = data.split(':')[0] - real_data = data.split(':')[1] + real_data = ''.join(data.split(':')[1:]) if 'user' in key or 'org' in key: - utils.just_append(user, real_data.strip()) + utils.just_append(user, real_data + "\n") else: - utils.just_append(repo, real_data.strip()) - utils.clean_up(repo) - utils.clean_up(user) + utils.just_append(repo, real_data + "\n") + # print(real_data) + try: + utils.clean_up(repo) + utils.clean_up(user) + except: + pass def routine(args): diff --git a/lib/alias/gitrecon b/lib/alias/gitrecon index b9b1828..7f900d3 100755 --- a/lib/alias/gitrecon +++ b/lib/alias/gitrecon @@ -31,12 +31,14 @@ def routine(args): datas = [target] if mode == 'user': - for target in datas: - cmd = f'{plugin}/go/gitrob -save {output} -threads 10 -github-access-token {key} {target}' + for data in datas: + cmd = f'{plugin}/go/gitrob -save {output} -threads 10 -github-access-token {key} {data}' + utils.print_info(f"Execute: {cmd}") os.system(cmd) else: - for target in datas: - cmd = f'trufflehog --regex --entropy=True {target} | tee {output}-trufflehog.txt' + for data in datas: + cmd = f'trufflehog --regex --entropy=True {data} | tee {output}-trufflehog.txt' + utils.print_info(f"Execute: {cmd}") os.system(cmd) diff --git a/lib/alias/portscan b/lib/alias/portscan index 5d3d421..aa66883 100755 --- a/lib/alias/portscan +++ b/lib/alias/portscan @@ -23,7 +23,7 @@ def routine(args): cmd = f'sudo masscan --rate 10000 -p0-65535 -iL {target} -oX {output}.xml --wait 0' else: cmd = f'sudo masscan --rate 10000 -p0-65535 {target} -oX {output}.xml --wait 0' - + os.system(cmd) utils.print_info(f"Execute: {cmd}") diff --git a/lib/core/dbutils.py b/lib/core/dbutils.py index db5f278..92c6b2d 100644 --- a/lib/core/dbutils.py +++ b/lib/core/dbutils.py @@ -107,7 +107,7 @@ def gen_default_config(config_path): configs.set('Enviroments', 'go_path', go_path) # set some tokens - github_api_key = utils.get_enviroment("GITROB_ACCESS_TOKEN") + github_api_key = utils.get_enviroment("GITHUB_API_KEY") slack_bot_token = utils.get_enviroment("SLACK_BOT_TOKEN") log_channel = utils.get_enviroment("LOG_CHANNEL") status_channel = utils.get_enviroment("STATUS_CHANNEL") @@ -148,7 +148,9 @@ def load_default_config(config_file=None): 'alias': key, 'desc': key, } - Configurations.objects.create(**item) + instance, created = Configurations.objects.get_or_create( + name=key) + Configurations.objects.filter(name=key).update(**item) return options diff --git a/lib/core/utils.py b/lib/core/utils.py index 7f8630c..e6d0b92 100644 --- a/lib/core/utils.py +++ b/lib/core/utils.py @@ -587,7 +587,12 @@ def just_write(filename, data, is_json=False, uniq=False): def clean_up(filename): if not filename: return False - data = just_read(filename, get_list=True) + + filename = os.path.normpath(filename) + if os.path.isfile(filename): + with open(filename, 'r') as f: + data = f.readlines() + real_data = list(set(data)) just_write(filename, "\n".join(real_data)) return True diff --git a/lib/mode/routine.py b/lib/mode/routine.py index 53c43e4..4b94c6e 100644 --- a/lib/mode/routine.py +++ b/lib/mode/routine.py @@ -10,7 +10,6 @@ from lib.mode import report def routine_handle(options): - # print(options) if options.get('mode') == "report": utils.print_load("Running with report mode") report.handle(options) diff --git a/lib/workflow/direct.py b/lib/workflow/direct.py index 6422aac..f4d0a60 100644 --- a/lib/workflow/direct.py +++ b/lib/workflow/direct.py @@ -342,7 +342,7 @@ class IPSpace: class GitScan: reports = [ { - "path": "$WORKSPACE/gitscan/$OUTPUT-repo.txt", + "path": "$WORKSPACE/gitscan/$OUTPUT-user.txt", "type": "bash", }, { @@ -355,22 +355,22 @@ class GitScan: 'general': [ { "banner": "parse input", - "cmd": "$ALIAS_PATH/format_input -i '$TARGET' -o '$WORKSPACE/gitscan/$OUTPUT'", - "output_path": "$WORKSPACE/gitscan/$OUTPUT-repo.txt", + "cmd": "$ALIAS_PATH/git_format -i '$RAW_TARGET' -o '$WORKSPACE/gitscan/$OUTPUT'", + "output_path": "$WORKSPACE/gitscan/$OUTPUT-user.txt", "std_path": "", "waiting": "first", }, { "requirement": "$WORKSPACE/gitscan/$OUTPUT-repo.txt", "banner": "Git recon repo", - "cmd": "$ALIAS_PATH/gitrecon -r $WORKSPACE/gitscan/$OUTPUT-repo.txt -o '$WORKSPACE/gitscan/$OUTPUT' -k '$GITHUB_API_KEY' -p '$PLUGIN_PATH' ", + "cmd": "$ALIAS_PATH/gitrecon -r $WORKSPACE/gitscan/$OUTPUT-repo.txt -o '$WORKSPACE/gitscan/$OUTPUT' -k '$GITHUB_API_KEY' -p '$PLUGINS_PATH' ", "output_path": "$WORKSPACE/gitscan/$TARGET-gitrob.txt", "std_path": "$WORKSPACE/gitscan/std-$TARGET-gitrob.std" }, { "requirement": "$WORKSPACE/gitscan/$OUTPUT-user.txt", "banner": "Git recon user", - "cmd": "$ALIAS_PATH/gitrecon -u $WORKSPACE/gitscan/$OUTPUT-repo.txt -o '$WORKSPACE/gitscan/$OUTPUT' -k '$GITHUB_API_KEY' -p '$PLUGIN_PATH' ", + "cmd": "$ALIAS_PATH/gitrecon -u $WORKSPACE/gitscan/$OUTPUT-user.txt -o '$WORKSPACE/gitscan/$OUTPUT' -k '$GITHUB_API_KEY' -p '$PLUGINS_PATH' ", "output_path": "$WORKSPACE/gitscan/$TARGET-gitrob.txt", "std_path": "$WORKSPACE/gitscan/std-$TARGET-gitrob.std" } diff --git a/scripts/clean.py b/scripts/clean.py index fd41a42..5d44d43 100755 --- a/scripts/clean.py +++ b/scripts/clean.py @@ -50,7 +50,7 @@ def load_default_config(): # load command from default workflow dbutils.internal_parse_commands() # create server config file - dbutils.load_default_config(confif_file='~/.osmedeus/server.conf') + dbutils.load_default_config(config_file='~/.osmedeus/server.conf') # Configurations.objects.create(**record) utils.print_good("Load config success") diff --git a/scripts/reload.py b/scripts/reload.py new file mode 100755 index 0000000..b5f8cb0 --- /dev/null +++ b/scripts/reload.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import os +import sys +import argparse +from pathlib import Path +import django + +# Load default path of extensions folder +# WARNING: change this if you custom something +BASE_DIR = Path(os.path.dirname( + os.path.dirname(os.path.abspath(__file__)))) +sys.path.append(str(BASE_DIR.joinpath('server'))) + +# loading django setup +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "rest.settings") +django.setup() + +from api.models import * + +from core import dbutils +from core import utils + + +# reload routine +def reload_routine(): + # load command from default workflow + dbutils.internal_parse_commands() + # create server config file + dbutils.load_default_config(config_file='~/.osmedeus/server.conf') + + +def main(args): + # workspace = args.workspace if args.workspace else None + utils.print_block("Reload routine and config for server", tag='RUN') + reload_routine() + + +parser = argparse.ArgumentParser(description="Reload routine for server") +parser.add_argument('-w', '--workspace', action='store', + dest='workspace', help='workspace') +args = parser.parse_args() +main(args) diff --git a/server/api/models.py b/server/api/models.py index f4f056a..ff690fd 100644 --- a/server/api/models.py +++ b/server/api/models.py @@ -9,7 +9,7 @@ from core import utils # store some configuration class Configurations(models.Model): - name = models.TextField(blank=False, default='') + name = models.TextField(unique=True, blank=False, default='') value = models.TextField(blank=False, default='') alias = models.TextField(default='') desc = models.TextField(default='')