Release v4.6.0

.github/FUNDING.yml

@@ -0,0 +1,12 @@
+# These are supported funding model platforms
+
+github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
+patreon: j3ssie
+open_collective: osmedeus
+ko_fi: j3ssie
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+custom: [ 'https://paypal.me/j3ssiejjj' ]

.gitignore

@@ -0,0 +1,5 @@
+.DS_STORE
+.idea
+out*
+osmedeus
+dist/*

LICENSE

@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) 2020 j3ssie
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

Makefile

@@ -0,0 +1,38 @@
+TARGET   ?= osmedeus
+GO       ?= go
+GOFLAGS  ?= 
+VERSION  := $(shell cat libs/version.go | grep 'VERSION =' | cut -d '"' -f 2)
+
+build:
+	go install
+	go build -ldflags="-s -w" -tags netgo -trimpath -buildmode=pie -o dist/$(TARGET)
+
+release:
+	go install
+	@echo "==> Clean up old builds"
+	rm -rf ./dist/* ~/myGit/premium-$(TARGET)-base/dist/* ~/org-$(TARGET)/$(TARGET)-base/dist/*
+	@echo "==> building binaries for for mac intel"
+	GOOS=darwin GOARCH=amd64 go build -ldflags="-s -w" -tags netgo -trimpath -buildmode=pie -o dist/$(TARGET)
+	zip -9 -j dist/$(TARGET)-macos-amd64.zip dist/$(TARGET) && rm -rf ./dist/$(TARGET)
+	@echo "==> building binaries for for mac M1 chip"
+	CGO_ENABLED=1 GOOS=darwin GOARCH=arm64 go build -ldflags="-s -w" -tags netgo -trimpath -buildmode=pie -o dist/$(TARGET)
+	zip -9 -j dist/$(TARGET)-macos-arm64.zip dist/$(TARGET)&& rm -rf ./dist/$(TARGET)
+	@echo "==> building binaries for linux intel build on mac"
+	GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -tags netgo -trimpath -buildmode=pie -o dist/$(TARGET)
+	zip -j dist/$(TARGET)-linux.zip dist/$(TARGET)&& rm -rf ./dist/$(TARGET)
+	cp dist/* ~/myGit/premium-$(TARGET)-base/dist/
+	cp dist/* ~/org-$(TARGET)/$(TARGET)-base/dist/
+	@echo "==> Generating metadata info"
+	$(TARGET) update --gen dist/public.json
+	mv dist/$(TARGET)-macos-amd64.zip dist/$(TARGET)-$(VERSION)-macos-amd64.zip
+	mv dist/$(TARGET)-macos-arm64.zip dist/$(TARGET)-$(VERSION)-macos-arm64.zip
+	mv dist/$(TARGET)-linux.zip dist/$(TARGET)-$(VERSION)-linux.zip
+run:
+	$(GO) $(GOFLAGS) run *.go
+
+fmt:
+	$(GO) $(GOFLAGS) fmt ./...; \
+	echo "Done."
+
+test:
+	$(GO) $(GOFLAGS) test ./... -v%

README.md

@@ -0,0 +1,157 @@
+# Osmedeus Core Engine
+
+<p align="center">
+  <a href="https://www.osmedeus.org"><img alt="Osmedeus" src="https://raw.githubusercontent.com/osmedeus/assets/main/logo-transparent.png" height="140" /></a>
+  <br />
+  <strong>Osmedeus - A Workflow Engine for Offensive Security</strong>
+
+  <p align="center">
+  <a href="https://docs.osmedeus.org/"><img src="https://img.shields.io/badge/Documentation-0078D4?style=for-the-badge&logo=GitBook&logoColor=39ff14&labelColor=black&color=black"></a>
+  <a href="https://docs.osmedeus.org/donation/"><img src="https://img.shields.io/badge/Sponsors-0078D4?style=for-the-badge&logo=GitHub-Sponsors&logoColor=39ff14&labelColor=black&color=black"></a>
+  <a href="https://twitter.com/OsmedeusEngine"><img src="https://img.shields.io/badge/%40OsmedeusEngine-0078D4?style=for-the-badge&logo=Twitter&logoColor=39ff14&labelColor=black&color=black"></a>
+  <a href="https://discord.gg/gy4SWhpaPU"><img src="https://img.shields.io/badge/Discord%20Server-0078D4?style=for-the-badge&logo=Discord&logoColor=39ff14&labelColor=black&color=black"></a>
+  <a href="https://github.com/j3ssie/osmedeus/releases"><img src="https://img.shields.io/github/release/j3ssie/osmedeus?style=for-the-badge&labelColor=black&color=2fc414&logo=Github"></a>
+  </p>
+</p>
+
+***
+
+## 🔥 What is Osmedeus?
+
+Osmedeus is a Workflow Engine for Offensive Security. It was designed to build a foundation with the capability and
+flexibility that allows you to build your own reconnaissance system and run it on a large number of targets.
+
+## 📖 Documentation & FAQ
+
+You can check out the documentation at [**docs.osmedeus.org**](https://docs.osmedeus.org) and the Frequently Asked
+Questions at [**here**](https://docs.osmedeus.org/faq) for more information.
+
+## 📦 Installation
+
+> NOTE that you need some essential tools like `curl, wget, git, zip` and login as **root** to start
+
+```bash
+bash <(curl -fsSL https://raw.githubusercontent.com/osmedeus/osmedeus-base/master/install.sh)
+```
+
+### Build the engine from the source
+
+Make sure you installed `golang >= v1.17`
+
+```bash
+go install -v github.com/j3ssie/osmedeus@latest
+```
+
+Check out [**this page**](https://docs.osmedeus.org/installation/) for more the install on other platforms and [**docker
+image**](https://docs.osmedeus.org/installation/using-docker/).
+
+## 🚀 Key Features of Osmedeus
+
+- [x] Significantly speed up your recon process
+- [x] Organize your scan results
+- [x] Efficiently to customize and optimize your recon process
+- [x] Seamlessly integrate with new public and private tools
+- [x] Easy to scale across large number of targets
+- [x] Easy to synchronize the results across many places
+
+## 💡 Usage
+
+```bash
+# Example Scan Commands:
+  ## Start a simple scan with default 'general' flow
+  osmedeus scan -t sample.com
+
+  ## Start a general scan but exclude some of the module
+  osmedeus scan -t sample.com -x screenshot -x spider
+
+  ## Start a scan directly with a module with inputs as a list of http domains like this https://sub.example.com
+  osmedeus scan -m content-discovery -t http-file.txt
+
+  ## Initiate the scan using a speed option other than the default setting
+  osmedeus scan -f vuln --tactic gently -t sample.com
+  osmedeus scan --threads-hold=10 -t sample.com
+  osmedeus scan -B 5 -t sample.com
+
+  ## Start a simple scan with other flow
+  osmedeus scan -f vuln -t sample.com
+  osmedeus scan -f extensive -t sample.com -t another.com
+  osmedeus scan -f urls -t list-of-urls.txt
+
+  ## Scan list of targets
+  osmedeus scan -T list_of_targets.txt
+  osmedeus scan -f vuln -T list-of-targets.txt
+
+  ## Performing static vulnerability scan and secret scan on a git repo
+  osmedeus scan -m repo-scan -t https://github.com/j3ssie/sample-repo
+  osmedeus scan -m repo-scan -t /tmp/source-code-folder
+  osmedeus scan -m repo-scan -T list-of-repo.txt
+
+  ## Scan for CIDR with file contains CIDR with the format '1.2.3.4/24'
+  osmedeus scan -f cidr -t list-of-ciders.txt
+  osmedeus scan -f cidr -t '1.2.3.4/24' # this will auto convert the single input to the file and run
+
+  ## Directly run on vuln scan and directory scan on list of domains
+  osmedeus scan -f domains -t list-of-domains.txt
+  osmedeus scan -f vuln-and-dirb -t list-of-domains.txt
+
+  ## Use a custom wordlist
+  osmedeus scan -t sample.com -p 'wordlists={{Data}}/wordlists/content/big.txt'
+
+  ## Use a custom wordlist
+  cat list_of_targets.txt | osmedeus scan -c 2
+
+  ## Start a normal scan and backup entire workflow folder to the backup folder
+  osmedeus scan --backup -f domains -t list-of-subdomains.txt
+
+  ## Start the scan with chunk inputs to review the output way more much faster
+  osmedeus scan --chunk --chunk-parts 20 -f cidr -t list-of-100-cidr.txt
+
+  ## Continuously run the scan on a target right after it finished
+  osmedeus utils cron --for --cmd 'osmedeus scan -t example.com'
+
+  ## Backing up all workspaces
+  ls ~/workspaces-osmedeus | osmedeus report compress
+
+
+# Scan Usage:
+  osmedeus scan -f [flowName] -t [target]
+  osmedeus scan -m [modulePath] -T [targetsFile]
+  osmedeus scan -f /path/to/flow.yaml -t [target]
+  osmedeus scan -m /path/to/module.yaml -t [target] --params 'port=9200'
+  osmedeus scan -m /path/to/module.yaml -t [target] -l /tmp/log.log
+  osmedeus scan --tactic aggressive -m module -t [target]
+  cat targets | osmedeus scan -f sample
+
+# Practical Scan Usage:
+  osmedeus scan -T list_of_targets.txt -W custom_workspaces
+  osmedeus scan -t target.com -w workspace_name --debug
+  osmedeus scan -f general -t sample.com
+  osmedeus scan --tactic aggressive -f general -t sample.com
+  osmedeus scan -f extensive -t sample.com -t another.com
+  cat list_of_urls.txt | osmedeus scan -f urls
+  osmedeus scan --threads-hold=15 -f cidr -t 1.2.3.4/24
+  osmedeus scan -m ~/.osmedeus/core/workflow/test/dirbscan.yaml -t list_of_urls.txt
+  osmedeus scan --wfFolder ~/custom-workflow/ -f your-custom-workflow -t list_of_urls.txt
+  osmedeus scan --chunk --chunk-part 40 -c 2 -f cidr -t list-of-cidr.txt
+
+💡 For full help message, please run: osmedeus --hh or osmedeus scan --hh
+📖 Documentation can be found here: https://docs.osmedeus.org
+```
+
+Check out [**this page**](https://docs.osmedeus.org/installation/usage/) for full usage and the [**Practical Usage**](https://docs.osmedeus.org/installation/practical-usage/) to see how to use Osmedeus in a practical way.
+
+## 💬 Community & Discussion
+
+Join Our Discord server [here](https://discord.gg/mtQG2FQsYA)
+
+## 💎 Donation & Sponsor
+
+<h3 align="center">
+ <img alt="Osmedeus" src="https://raw.githubusercontent.com/osmedeus/assets/main/premium-package.gif" />
+
+ <p align="center"> Check out for a couple of <strong><a href="https://docs.osmedeus.org/donation/">donation methods here</a></strong> to get a <strong><a href="https://docs.osmedeus.org/premium/">premium package</a></strong><p>
+</h3>
+
+## License
+
+`Osmedeus` is made with ♥ by [@j3ssiejjj](https://twitter.com/j3ssiejjj) and it is released under the MIT license.

cmd/cloud.go

@@ -0,0 +1,185 @@
+package cmd
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path"
+
+	"github.com/fatih/color"
+	"github.com/j3ssie/osmedeus/core"
+	"github.com/j3ssie/osmedeus/distribute"
+	"github.com/j3ssie/osmedeus/libs"
+	"github.com/j3ssie/osmedeus/utils"
+	"github.com/spf13/cobra"
+	"golang.org/x/text/cases"
+	"golang.org/x/text/language"
+)
+
+func init() {
+	var cloudCmd = &cobra.Command{
+		Use:   "cloud",
+		Short: "Perform a scan using the distributed cloud mode",
+		Long:  core.Banner(),
+		RunE:  runCloud,
+	}
+
+	// core options
+	cloudCmd.Flags().StringVarP(&options.Cloud.Module, "module", "m", "", "module name for running")
+	cloudCmd.Flags().StringVarP(&options.Cloud.Flow, "flow", "f", "general", "Flow name for running (default: general)")
+	cloudCmd.Flags().StringVarP(&options.Cloud.Workspace, "workspace", "w", "", "Name of workspace (default is same as target)")
+	cloudCmd.Flags().StringSliceVarP(&options.Cloud.Params, "params", "p", []string{}, "Custom params -p='foo=bar' (Multiple -p flags are accepted)")
+
+	// chunk inputs
+	cloudCmd.Flags().BoolVar(&options.Cloud.EnablePrivateIP, "privateIP", false, "Enable Private IP")
+	cloudCmd.Flags().BoolVar(&options.Cloud.TargetAsFile, "as-file", false, "Run target as file (use -T targets.txt file instead of -t targets.txt at cloud instance)")
+	cloudCmd.Flags().StringVar(&options.Cloud.LocalSyncFolder, "rfolder", "", "Remote Folder to sync back to local")
+
+	// commands on cloud
+	cloudCmd.Flags().IntVar(&options.Cloud.Threads, "cloud-threads", 1, "Concurrency level on remote cloud")
+	cloudCmd.Flags().StringVar(&options.Cloud.Extra, "extra", "", "append raw command after the command builder")
+	cloudCmd.Flags().StringVar(&options.Cloud.RawCommand, "cmd", "", "specific raw command and override everything (eg: --cmd 'curl {{Target}}')")
+	cloudCmd.Flags().StringVar(&options.Cloud.ClearTime, "clear", "10m", "time to wait before next clear check")
+	cloudCmd.Flags().StringVar(&options.Cloud.TempTarget, "tempTargets", "/tmp/osm-tmp-inputs/", "Temp Folder to store targets file")
+
+	// mics option
+	cloudCmd.Flags().BoolVar(&options.Cloud.EnableSyncWorkflow, "sync-workflow", false, "Enable Sync Workflow folder to remote machine first before starting the scan")
+	cloudCmd.Flags().BoolVarP(&options.Cloud.CopyWorkspaceToGit, "gws", "G", false, "Enable Copy Workspace to Git (run -f sync after done)")
+	cloudCmd.Flags().BoolVarP(&options.Cloud.DisableLocalSync, "no-lsync", "z", false, "Disable sync back data to local machine")
+	cloudCmd.Flags().BoolVar(&options.Cloud.BackgroundRun, "bg", false, "Send command to instance without checking if process is done or not")
+	cloudCmd.Flags().BoolVar(&options.Cloud.EnableTerraform, "tf", false, "Use terraform to create cloud instance")
+	cloudCmd.Flags().BoolVar(&options.Cloud.NoDelete, "no-del", false, "Don't delete instance after done (you can run 'osmedeus provider health' to clean up later)")
+	cloudCmd.Flags().BoolVar(&options.Cloud.IgnoreProcess, "no-ps", false, "Disable checking process on remote machine")
+	cloudCmd.Flags().IntVar(&options.Cloud.Retry, "retry", 10, "Number of retry when command is error")
+	cloudCmd.SetHelpFunc(CloudHelp)
+	RootCmd.AddCommand(cloudCmd)
+	cloudCmd.PreRun = func(cmd *cobra.Command, args []string) {
+		if options.FullHelp {
+			cmd.Help()
+			os.Exit(0)
+		}
+	}
+}
+
+func runCloud(cmd *cobra.Command, _ []string) error {
+	// DBInit()
+	utils.GoodF("%v %v by %v", cases.Title(language.Und, cases.NoLower).String(libs.BINARY), libs.VERSION, color.HiMagentaString(libs.AUTHOR))
+	utils.InforF("Storing the log file to: %v", color.CyanString(options.LogFile))
+
+	// parse some arguments
+	threads, _ := cmd.Flags().GetInt("thread")
+	if threads > 1 || options.Cloud.Threads <= 1 {
+		options.Cloud.Threads = threads
+	}
+
+	// get pre run commands
+	getPreRun(&options)
+
+	// change targets list if chunk mode enable
+	if options.Cloud.EnableChunk {
+		utils.InforF("Running cloud scan in chunk mode")
+		for _, target := range options.Scan.Inputs {
+			chunkTargets := HandleChunksInputs(target)
+			if len(chunkTargets) == 0 {
+				continue
+			}
+
+			distribute.InitCloud(options, chunkTargets)
+			// remove chunk inputs
+			utils.DebugF("Remove chunk inputs file")
+			for _, ctarget := range chunkTargets {
+				os.RemoveAll(ctarget)
+			}
+		}
+		return nil
+	}
+
+	// @NOTE: pro-tips
+	if options.Concurrency > 1 && len(options.Scan.Inputs) == 1 {
+		if !utils.FileExists(options.Scan.Inputs[0]) {
+			utils.WarnF("You're using %v in cloud scan but your input %v is just a single domain", color.HiMagentaString(`'-c %v'`, options.Concurrency), color.HiMagentaString(`'-t %v'`, options.Scan.Inputs[0]))
+			utils.WarnF("Consider running: osmedeus cloud -c 5 -T list-of-targets.txt")
+		}
+	}
+
+	distribute.InitCloud(options, options.Scan.Inputs)
+	return nil
+}
+
+// HandleChunksInputs split the inputs to multiple file first
+func HandleChunksInputs(target string) []string {
+	var chunkTargets []string
+	utils.MakeDir(options.Cloud.ChunkInputs)
+
+	if !utils.FileExists(target) {
+		utils.ErrorF("error to split input file: %v", target)
+		return chunkTargets
+	}
+
+	if options.Cloud.NumberOfParts == 0 {
+		options.Cloud.NumberOfParts = options.Concurrency
+	}
+
+	utils.DebugF("Splitting %v to %v part", target, options.Cloud.NumberOfParts)
+	rawChunks, err := utils.SplitLineChunks(target, options.Cloud.NumberOfParts)
+	if err != nil || len(rawChunks) == 0 {
+		utils.ErrorF("error to split input file: %v", target)
+		return chunkTargets
+	}
+	fp, err := os.Open(target)
+	if err != nil {
+		utils.ErrorF("error to open input file: %v", target)
+		return chunkTargets
+	}
+	for index, offset := range rawChunks {
+		targetName := fmt.Sprintf("%s-chunk-%v", utils.CleanPath(target), index)
+		targetName = path.Join(options.Cloud.ChunkInputs, targetName)
+
+		sectionReader := io.NewSectionReader(fp, offset.Start, offset.Stop-offset.Start)
+		targetFile, err := os.OpenFile(targetName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
+		if err != nil {
+			utils.ErrorF("error when create chunk file: %v", target)
+			continue
+		}
+
+		_, err = io.Copy(targetFile, sectionReader)
+		if err != nil {
+			utils.ErrorF("error to read chunk file: %s", err)
+			continue
+		}
+		targetFile.Close()
+		chunkTargets = append(chunkTargets, targetName)
+	}
+
+	return chunkTargets
+}
+
+func getPreRun(options *libs.Options) {
+	if options.Cloud.Module != "" {
+		module := core.DirectSelectModule(*options, options.Cloud.Module)
+		if module == "" {
+			utils.ErrorF("Error to select module: %s", options.Cloud.Module)
+			return
+		}
+		parsedModule, err := core.ParseModules(module)
+		if err == nil {
+			options.Cloud.RemotePreRun = parsedModule.RemotePreRun
+			options.Cloud.LocalPostRun = parsedModule.LocalPostRun
+			options.Cloud.LocalPreRun = parsedModule.LocalPreRun
+			options.Cloud.LocalSteps = parsedModule.LocalSteps
+		}
+		return
+	}
+
+	if options.Cloud.Flow != "" {
+		flows := core.SelectFlow(options.Cloud.Flow, *options)
+		for _, flow := range flows {
+			parseFlow, err := core.ParseFlow(flow)
+			if err == nil {
+				options.Cloud.RemotePreRun = parseFlow.RemotePreRun
+				options.Cloud.LocalPostRun = parseFlow.LocalPostRun
+				options.Cloud.LocalPreRun = parseFlow.LocalPreRun
+			}
+		}
+	}
+}

cmd/config.go

@@ -0,0 +1,100 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"sort"
+	"strings"
+
+	"github.com/fatih/color"
+	"github.com/j3ssie/osmedeus/execution"
+
+	"github.com/j3ssie/osmedeus/core"
+	"github.com/j3ssie/osmedeus/utils"
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	var configCmd = &cobra.Command{
+		Use:   "config",
+		Short: "Do some configuration from CLI",
+		Long:  core.Banner(),
+		RunE:  runConfig,
+	}
+
+	configCmd.Flags().StringP("action", "a", "", "Action")
+	configCmd.Flags().String("pluginsRepo", "git@gitlab.com:j3ssie/osmedeus-plugins.git", "Osmedeus Plugins repository")
+	// for cred action
+	configCmd.Flags().String("user", "", "Username")
+	configCmd.Flags().String("pass", "", "Password")
+	configCmd.Flags().StringP("workspace", "w", "", "Name of workspace")
+	configCmd.SetHelpFunc(ConfigHelp)
+	RootCmd.AddCommand(configCmd)
+}
+
+func runConfig(cmd *cobra.Command, args []string) error {
+	sort.Strings(args)
+	action, _ := cmd.Flags().GetString("action")
+	workspace, _ := cmd.Flags().GetString("workspace")
+
+	// backward compatible
+	if action == "" && len(args) > 0 {
+		action = args[0]
+	}
+
+	switch action {
+	case "init":
+		if utils.FolderExists(fmt.Sprintf("%vcore", options.Env.RootFolder)) {
+			utils.GoodF("Look like you got properly setup.")
+		}
+		break
+	case "cred":
+		username, _ := cmd.Flags().GetString("user")
+		password, _ := cmd.Flags().GetString("pass")
+		utils.GoodF("Create new credentials %v:%v \n", username, password)
+		break
+
+	case "reload":
+		fmt.Println("💬 Reload the configuration will replace current settings with new ones based on the current environment")
+		var input string
+		fmt.Printf(color.HiRedString("🌀 Do you want to proceed? (y/N): "))
+		fmt.Scan(&input)
+		input = strings.ToLower(input)
+		if input == "yes" || input == "y" {
+			utils.InforF("Delete current config and generate a new one")
+			os.Remove(options.ConfigFile)
+			os.Remove(options.TokenConfigFile)
+			core.InitConfig(&options)
+			core.ParsingConfig(&options)
+		}
+		break
+
+	case "delete", "del":
+		options.Scan.Input = workspace
+		options.Scan.ROptions = core.ParseInput(options.Scan.Input, options)
+		utils.InforF("Delete Workspace: %v", options.Scan.ROptions["Workspace"])
+		os.RemoveAll(options.Scan.ROptions["Output"])
+		break
+
+	case "pull":
+		for repo := range options.Storages {
+			execution.PullResult(repo, options)
+		}
+		break
+
+	case "set":
+		core.SetTactic(&options)
+		break
+	case "update":
+		core.Update(options)
+		break
+
+	case "clean", "cl", "c":
+		break
+	default:
+		utils.ErrorF("Unknown action: %v", color.HiRedString(action))
+		fmt.Println(ConfigUsage())
+	}
+
+	return nil
+}

cmd/exec.go

@@ -0,0 +1,56 @@
+package cmd
+
+import (
+	"os"
+
+	"github.com/j3ssie/osmedeus/core"
+	"github.com/j3ssie/osmedeus/utils"
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	var execCmd = &cobra.Command{
+		Use:   "exec",
+		Short: "Execute inline osmedeus scripts",
+		Long:  core.Banner(),
+		RunE:  runExec,
+	}
+
+	execCmd.Flags().String("script", "", "Scripts to run (Multiple -s flags are accepted)")
+	execCmd.Flags().StringP("scriptFile", "S", "", "File contain list of scripts")
+	RootCmd.AddCommand(execCmd)
+}
+
+func runExec(cmd *cobra.Command, _ []string) error {
+	script, _ := cmd.Flags().GetString("script")
+	scriptFile, _ := cmd.Flags().GetString("scriptFile")
+
+	var scripts []string
+	if script != "" {
+		scripts = append(scripts, script)
+	}
+	if scriptFile != "" {
+		moreScripts := utils.ReadingFileUnique(scriptFile)
+		if len(moreScripts) > 0 {
+			scripts = append(scripts, moreScripts...)
+		}
+	}
+
+	if len(scripts) == 0 {
+		utils.ErrorF("No scripts provided")
+		os.Exit(0)
+	}
+	runner, _ := core.InitRunner("example.com", options)
+
+	for _, t := range options.Scan.Inputs {
+		// start to run scripts
+		options.Scan.ROptions = core.ParseInput(t, options)
+		for _, rscript := range scripts {
+			script = core.ResolveData(rscript, options.Scan.ROptions)
+			utils.InforF("Script: %v", script)
+			runner.RunScript(script)
+		}
+	}
+
+	return nil
+}

cmd/health.go

@@ -0,0 +1,283 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"sort"
+
+	"github.com/fatih/color"
+	"github.com/j3ssie/osmedeus/core"
+	"github.com/j3ssie/osmedeus/execution"
+	"github.com/j3ssie/osmedeus/libs"
+	"github.com/j3ssie/osmedeus/utils"
+	"github.com/olekukonko/tablewriter"
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	var healthCmd = &cobra.Command{
+		Use:     "health",
+		Aliases: []string{"hea", "heal", "health", "healht"},
+		Short:   "Run diagnostics to check configurations",
+		Long:    core.Banner(),
+		RunE:    runHealth,
+	}
+	RootCmd.AddCommand(healthCmd)
+	healthCmd.PreRun = func(cmd *cobra.Command, args []string) {
+		if options.FullHelp {
+			cmd.Help()
+			os.Exit(0)
+		}
+	}
+}
+
+func runHealth(_ *cobra.Command, args []string) error {
+	if options.PremiumPackage {
+		fmt.Printf("💠 Osmedeus Premium %s: Run diagnostics to ensure that everything is in arrange.\n", libs.VERSION)
+	} else {
+		fmt.Printf("🚀 Osmedeus %s: Run diagnostics to ensure that everything is in arrange.\n", libs.VERSION)
+	}
+
+	sort.Strings(args)
+	var err error
+	for _, arg := range args {
+		switch arg {
+		case "store", "git", "storages", "stora":
+			err = checkStorages()
+		case "cloud", "dist", "provider":
+			err = checkCloud()
+		case "all", "a", "full":
+			err = checkStorages()
+			if err != nil {
+				fmt.Println(color.YellowString("⚠️️ There is might be something wrong with your storages: %v\n", err))
+			}
+			err = checkCloud()
+			if err != nil {
+				fmt.Println(color.YellowString("%s If you install osmedeus on a single machine then it's okay to ignore the cloud setup\n", "[!] Cloud config setup incorrectly."))
+			}
+			err = generalCheck()
+			if err != nil {
+				fmt.Printf("‼️ There is might be something wrong with your setup: %v\n", color.HiRedString("%v", err))
+				return nil
+			}
+			break
+		}
+		if err != nil {
+			fmt.Println(color.YellowString("⚠️️ There is might be something wrong with your cloud or storages setup: %v\n", err))
+			return nil
+		}
+	}
+	if len(args) > 0 {
+		return nil
+	}
+
+	if err = generalCheck(); err != nil {
+		fmt.Printf("‼️ There is might be something wrong with your setup: %v\n", err)
+		return nil
+	}
+
+	if err = listFlows(); err != nil {
+		fmt.Printf("‼️ There is might be something wrong with your workflow setup: %v\n", err)
+		return nil
+	}
+
+	if err = listDefaultModules(); err != nil {
+		fmt.Printf("‼️ There is might be something wrong with your workflow setup: %v\n", err)
+		return nil
+	}
+	fmt.Printf(color.GreenString("\n🦾 Everything is in order. Happy Hacking 🦾\n"))
+	return nil
+}
+
+func checkCloud() error {
+	// check packer program
+	if _, err := utils.RunCommandWithErr("packer -h"); err != nil {
+		if _, err := utils.RunCommandWithErr(fmt.Sprintf("%s -h", path.Join(options.Env.BinariesFolder, "packer"))); err != nil {
+			color.Red("[-] Packer program setup incorrectly")
+			return fmt.Errorf("error checking core programs: %v", fmt.Sprintf("%s -h", path.Join(options.Env.BinariesFolder, "packer")))
+		}
+	}
+
+	// check config files
+	if !utils.FileExists(options.CloudConfigFile) {
+		return fmt.Errorf("distributed cloud config doesn't exist: %v", path.Join(options.Env.CloudConfigFolder, "provider.yaml"))
+	}
+	if utils.DirLength(path.Join(options.Env.CloudConfigFolder, "providers")) == 0 {
+		return fmt.Errorf("providers file doesn't exist: %v", path.Join(options.Env.CloudConfigFolder, "providers"))
+	}
+
+	// check SSH Keys
+	if !utils.FileExists(options.Cloud.SecretKey) {
+		keysDir := path.Dir(options.Cloud.SecretKey)
+		os.RemoveAll(keysDir)
+		utils.MakeDir(keysDir)
+		utils.DebugF("Generate SSH Key at: %v", options.Cloud.SecretKey)
+		if _, err := utils.RunCommandWithErr(fmt.Sprintf(`ssh-keygen -t ed25519 -f %s -q -N ''`, options.Cloud.SecretKey)); err != nil {
+			color.Red("[-] error generated SSH Key for cloud config at: %v", options.Cloud.SecretKey)
+			return fmt.Errorf("[-] error generated SSH Key for cloud config at: %v", options.Cloud.SecretKey)
+		}
+	}
+	if !utils.FileExists(options.Cloud.PublicKey) {
+		return fmt.Errorf("providers SSH Key missing: %v", options.Cloud.PublicKey)
+	}
+
+	fmt.Printf("[+] Health Check Cloud Config: %s\n", color.GreenString("✔"))
+	return nil
+}
+
+func checkStorages() error {
+	utils.DebugF("Checking storages setup")
+	if !execution.ValidGitURL(options.Storages["summary_repo"]) {
+		return fmt.Errorf("invalid git summary: %v", options.Storages["summary_repo"])
+	}
+
+	utils.DebugF("Check if your summary directory is exist or not: %v", options.Env.StoragesFolder)
+	if utils.DirLength(options.Env.StoragesFolder) < 1 {
+		return fmt.Errorf("storages folder doesn't exist: %v", options.Env.StoragesFolder)
+	}
+
+	utils.DebugF("Check the secret key for git usage: %v", options.Storages["secret_key"])
+	if !utils.FileExists(options.Storages["secret_key"]) {
+		return fmt.Errorf("secret key for git command doesn't exist: %v", options.Storages["secret_key"])
+	}
+
+	fmt.Printf("[+] Health Check Storages Config: %s\n", color.GreenString("✔"))
+	return nil
+}
+
+func generalCheck() error {
+	exist := utils.FolderExists(options.Env.BaseFolder)
+	if !exist {
+		color.Red("[-] Core folder setup incorrect: %v", options.Env.BaseFolder)
+		return fmt.Errorf("error running diagnostics")
+	}
+
+	// check core programs
+	var err error
+	// if _, err = utils.RunCommandWithErr("jaeles -h"); err != nil {
+	// 	color.Red("[-] Core program setup incorrectly")
+	// 	return fmt.Errorf("error checking core programs: %v", "jaeles")
+	// }
+	if _, err = utils.RunCommandWithErr("timeout --help"); err != nil {
+		color.Red("[-] Core program setup incorrectly")
+		return fmt.Errorf("error checking core programs: %v", "timeout")
+	}
+	if _, err = utils.RunCommandWithErr("amass -h"); err != nil {
+		color.Red("[-] Core program setup incorrectly")
+		return fmt.Errorf("error checking core programs: %v", "amass")
+	}
+	_, err = utils.RunCommandWithErr(fmt.Sprintf("%s -h", path.Join(options.Env.BinariesFolder, "httprobe")))
+	if err != nil {
+		color.Red("[-] Core program setup incorrectly")
+		return fmt.Errorf("error checking core programs: %v", fmt.Sprintf("%s -h", path.Join(options.Env.BinariesFolder, "httprobe")))
+	}
+	fmt.Printf("[+] Health Check Core Programs: %s\n", color.GreenString("✔"))
+
+	// Check core signatures
+	okVuln := false
+	// if utils.DirLength("~/.jaeles/base-signatures/") > 0 || utils.DirLength("~/pro-signatures/") > 0 {
+	// 	okVuln = true
+	// }
+
+	if utils.DirLength("~/nuclei-templates") > 0 {
+		okVuln = true
+	}
+
+	if okVuln {
+		fmt.Printf("[+] Health Check Vulnerability scanning config: %s\n", color.GreenString("✔"))
+	} else {
+		color.Red("vulnerability scanning config setup incorrectly")
+		return fmt.Errorf("vulnerability scanning config setup incorrectly")
+	}
+
+	// check data folder
+	if utils.FolderExists(options.Env.DataFolder) {
+		fmt.Printf("[+] Health Check Data Config: %s\n", color.GreenString("✔"))
+	} else {
+		color.Red("[-] Data setup incorrectly: %v", options.Env.DataFolder)
+		return fmt.Errorf("[-] Data setup incorrectly: %v", options.Env.DataFolder)
+	}
+	return nil
+}
+
+func listFlows() error {
+	flows := core.ListFlow(options)
+	if len(flows) == 0 {
+		color.Red("[-] Error to list workflows: %s", options.Env.WorkFlowsFolder)
+		return fmt.Errorf("[-] Error to list workflows: %s", options.Env.WorkFlowsFolder)
+	}
+	fmt.Printf("[+] Health Check Workflows: %s\n", color.GreenString("✔"))
+	if options.PremiumPackage {
+		fmt.Printf("💎 Making use of the premium workflow\n")
+	}
+
+	var content [][]string
+	for _, flow := range flows {
+		parsedFlow, err := core.ParseFlow(flow)
+		if err != nil {
+			utils.ErrorF("Error parsing flow: %v", flow)
+			continue
+		}
+
+		if parsedFlow.SkipIndexed {
+			continue
+		}
+
+		row := []string{
+			parsedFlow.Name, parsedFlow.Desc,
+		}
+
+		content = append(content, row)
+	}
+	fmt.Printf("\nFound %v available workflows at: %s \n\n", color.HiGreenString("%v", len(content)), color.HiCyanString(options.Env.WorkFlowsFolder))
+
+	table := tablewriter.NewWriter(os.Stdout)
+	table.SetAutoFormatHeaders(false)
+	table.SetHeader([]string{"Flow Name", "Description"})
+	table.SetBorders(tablewriter.Border{Left: true, Top: true, Right: true, Bottom: true})
+	table.SetColWidth(120)
+	table.AppendBulk(content) // Add Bulk Data
+	table.Render()
+
+	h := color.HiCyanString("\nUsage:\n")
+	h += color.HiGreenString(" osmedeus scan -f %v", color.HiMagentaString("[flowName]")) + color.HiGreenString(" -t ") + color.HiMagentaString("[target]") + "\n"
+	fmt.Printf(h)
+	return nil
+}
+
+func listDefaultModules() error {
+	defaultModule := path.Join(options.Env.WorkFlowsFolder, "default-modules")
+	modules := core.DefaultWorkflows(options)
+
+	if len(modules) == 0 {
+		return fmt.Errorf("[-] Error to list default modules: %s", defaultModule)
+	}
+
+	var content [][]string
+	for _, flow := range modules {
+		parsedModule, err := core.ParseModules(flow)
+		if err != nil {
+			utils.ErrorF("Error parsing flow: %v", flow)
+			continue
+		}
+		row := []string{
+			parsedModule.Name, parsedModule.Desc,
+		}
+		content = append(content, row)
+	}
+	fmt.Printf("\nFound %v default modules at: %s \n\n", color.HiGreenString("%v", len(content)), color.HiCyanString(defaultModule))
+
+	table := tablewriter.NewWriter(os.Stdout)
+	table.SetAutoFormatHeaders(false)
+	table.SetHeader([]string{"Module Name", "Description"})
+	table.SetBorders(tablewriter.Border{Left: true, Top: true, Right: true, Bottom: true})
+	table.SetColWidth(120)
+	table.AppendBulk(content) // Add Bulk Data
+	table.Render()
+
+	h := color.HiCyanString("\nModule Usage:\n")
+	h += color.HiGreenString(" osmedeus scan -m %v", color.HiMagentaString("[moduleName]")) + color.HiGreenString(" -t ") + color.HiMagentaString("[target]") + "\n\n"
+	fmt.Printf(h)
+	return nil
+}

cmd/provider.go

@@ -0,0 +1,266 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"sync"
+	"time"
+
+	"github.com/j3ssie/osmedeus/core"
+	"github.com/j3ssie/osmedeus/distribute"
+	"github.com/j3ssie/osmedeus/provider"
+	"github.com/j3ssie/osmedeus/utils"
+	"github.com/olekukonko/tablewriter"
+	"github.com/panjf2000/ants"
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	var providerCmd = &cobra.Command{
+		Use:     "provider",
+		Aliases: []string{"provide", "pro"},
+		Short:   "Cloud utils for Distributed Mode",
+		Long:    core.Banner(),
+		RunE:    runProvider,
+	}
+
+	providerCmd.PersistentFlags().StringVar(&options.Cloud.RawCommand, "cmd", "", "raw command")
+	providerCmd.PersistentFlags().BoolVar(&options.Cloud.CheckingLimit, "check", false, "Only check for limit of config")
+	providerCmd.PersistentFlags().StringVar(&options.Cloud.InstanceName, "name", "", "override instance name")
+	providerCmd.PersistentFlags().BoolVar(&options.Cloud.BackgroundRun, "bg", false, "Send command to instance and run it in background")
+	providerCmd.PersistentFlags().BoolVar(&options.Cloud.IgnoreConfigFile, "ic", false, "Ignore token in the config file")
+	providerCmd.PersistentFlags().IntVar(&options.Cloud.Retry, "retry", 10, "Number of retry when command is error")
+	providerCmd.PersistentFlags().StringSlice("id", []string{}, "Instance IDs that will be delete")
+	providerCmd.Flags().StringVar(&options.Cloud.ClearTime, "clear", "10m", "time to wait before next clear check")
+	providerCmd.PersistentFlags().BoolVar(&options.Cloud.ForEverHealthCheck, "for", false, "Continuesly running the health check forever")
+
+	var providerWizard = &cobra.Command{
+		Use:     "wizard",
+		Aliases: []string{"wi", "wiz", "wizazrd"},
+		Short:   "Start a cloud config wizard",
+		Long:    core.Banner(),
+		RunE:    runCloudInit,
+	}
+	providerWizard.PersistentFlags().BoolVar(&options.Cloud.AddNewProvider, "add", false, "Open wizard to add new provider only")
+
+	var providerBuild = &cobra.Command{
+		Use:     "build",
+		Aliases: []string{"buil"},
+		Short:   "Build snapshot image",
+		Long:    core.Banner(),
+		RunE:    runProviderBuild,
+	}
+	var providerCreate = &cobra.Command{
+		Use:     "create",
+		Aliases: []string{"cre"},
+		Short:   "Create cloud instance based on image",
+		Long:    core.Banner(),
+		RunE:    runProviderCreate,
+	}
+
+	var providerHealth = &cobra.Command{
+		Use:     "health",
+		Aliases: []string{"hea", "heal", "health", "healht"},
+		Short:   "Conduct a health assessment on cloud instances that are currently operational",
+		Long:    core.Banner(),
+		RunE:    runCloudHealth,
+	}
+
+	var providerValidate = &cobra.Command{
+		Use:     "validate",
+		Aliases: []string{"val"},
+		Short:   "Run validate of the existing cloud configs",
+		Long:    core.Banner(),
+		RunE:    runProviderValidate,
+	}
+	var providerList = &cobra.Command{
+		Use:     "list",
+		Aliases: []string{"ls"},
+		Short:   "List all running instances",
+		Long:    core.Banner(),
+		RunE:    runProviderListing,
+	}
+	var providerDel = &cobra.Command{
+		Use:     "delete",
+		Aliases: []string{"del"},
+		Short:   "Delete instances by id",
+		Long:    core.Banner(),
+		RunE:    runProviderDelete,
+	}
+	var providerClear = &cobra.Command{
+		Use:     "clear",
+		Aliases: []string{"clea", "clr"},
+		Short:   "Clear all instances in the instances folders",
+		Long:    core.Banner(),
+		RunE:    runProviderClear,
+	}
+	providerCmd.AddCommand(providerWizard)
+	providerCmd.AddCommand(providerList)
+	providerCmd.AddCommand(providerDel)
+	providerCmd.AddCommand(providerValidate)
+	providerCmd.AddCommand(providerHealth)
+	providerCmd.AddCommand(providerCreate)
+	providerCmd.AddCommand(providerBuild)
+	providerCmd.AddCommand(providerClear)
+	providerCmd.SetHelpFunc(CloudHelp)
+	RootCmd.AddCommand(providerCmd)
+	providerCmd.PreRun = func(cmd *cobra.Command, args []string) {
+		if options.FullHelp {
+			cmd.Help()
+			os.Exit(0)
+		}
+	}
+}
+
+func runCloudHealth(_ *cobra.Command, _ []string) error {
+	distribute.CheckingCloudInstance(options)
+	if options.Cloud.ForEverHealthCheck {
+		for {
+			distribute.CheckingCloudInstance(options)
+			waitTime := utils.CalcTimeout(options.Cloud.ClearTime)
+			time.Sleep(time.Duration(waitTime) * time.Second)
+		}
+	}
+
+	return nil
+}
+
+func runProviderClear(_ *cobra.Command, _ []string) error {
+	distribute.ClearAllInstances(options)
+	return nil
+}
+
+func runCloudInit(_ *cobra.Command, _ []string) error {
+	// interactive mode to show config file here
+	distribute.InitCloudSetup(options)
+	return nil
+}
+
+func runProvider(_ *cobra.Command, args []string) error {
+	if len(args) == 0 {
+		fmt.Println(CloudUsage())
+	}
+	return nil
+}
+
+func runProviderBuild(_ *cobra.Command, _ []string) error {
+	options.Cloud.OnlyCreateDroplet = true
+	options.Cloud.ReBuildBaseImage = true
+
+	// building multiple tokens
+	options.Cloud.TokensFile = utils.NormalizePath(options.Cloud.TokensFile)
+	if options.Cloud.TokensFile != "" {
+		tokens := utils.ReadingFileUnique(options.Cloud.TokensFile)
+		if len(tokens) == 0 {
+			utils.ErrorF("token file not found: %v", options.Cloud.TokensFile)
+			return nil
+		}
+
+		var wg sync.WaitGroup
+		p, _ := ants.NewPoolWithFunc(options.Concurrency, func(i interface{}) {
+			lOptions := options
+			lOptions.Cloud.Token = i.(string)
+
+			distribute.InitCloud(lOptions, lOptions.Scan.Inputs)
+			wg.Done()
+		}, ants.WithPreAlloc(true))
+		defer p.Release()
+
+		for _, token := range tokens {
+			wg.Add(1)
+			_ = p.Invoke(token)
+		}
+		wg.Wait()
+		return nil
+
+	}
+
+	distribute.InitCloud(options, options.Scan.Inputs)
+	return nil
+}
+
+func runProviderCreate(_ *cobra.Command, _ []string) error {
+	options.Cloud.OnlyCreateDroplet = true
+	if len(options.Scan.Inputs) == 0 {
+		options.Scan.Inputs = append(options.Scan.Inputs, utils.RandomString(4))
+	}
+
+	distribute.InitCloud(options, options.Scan.Inputs)
+	return nil
+}
+
+func runProviderValidate(_ *cobra.Command, _ []string) error {
+	cloudValidate()
+	return nil
+}
+
+func runProviderListing(_ *cobra.Command, _ []string) error {
+	options.Cloud.BackgroundRun = true
+	utils.InforF("Listing all instances in the cloud provider(s) ...")
+	cloudRunners := distribute.GetClouds(options)
+	cloudListing(cloudRunners)
+	return nil
+}
+
+func runProviderDelete(cmd *cobra.Command, _ []string) error {
+	options.Cloud.OnlyCreateDroplet = true
+	options.Cloud.BackgroundRun = true
+	cloudRunners := distribute.GetClouds(options)
+	InstanceIDs, _ := cmd.Flags().GetStringSlice("id")
+
+	for _, InstanceID := range InstanceIDs {
+		for _, cloudRunner := range cloudRunners {
+			cloudRunner.Provider.DeleteInstance(InstanceID)
+		}
+	}
+
+	cloudListing(cloudRunners)
+	return nil
+}
+
+func cloudListing(cloudRunners []distribute.CloudRunner) {
+	var content [][]string
+	for _, cloudRunner := range cloudRunners {
+		cloudRunner.Provider.Action(provider.ListInstance)
+		for _, instance := range cloudRunner.Provider.Instances {
+			row := []string{
+				cloudRunner.Provider.ProviderName,
+				cloudRunner.Provider.RedactedToken,
+				instance.InstanceID,
+				instance.InstanceName,
+				instance.IPAddress,
+			}
+			content = append(content, row)
+		}
+	}
+	table := tablewriter.NewWriter(os.Stderr)
+	table.SetAutoFormatHeaders(false)
+	table.SetHeader([]string{"Provider", "Token", "Instance ID", "Instance Name", "IP Address"})
+	table.SetBorders(tablewriter.Border{Left: true, Top: false, Right: true, Bottom: false})
+	table.SetCenterSeparator("|")
+	table.AppendBulk(content) // Add Bulk Data
+	table.Render()
+}
+
+func cloudValidate() {
+	cloudRunners := distribute.GetClouds(options)
+
+	var content [][]string
+	for _, cloudRunner := range cloudRunners {
+		row := []string{
+			cloudRunner.Provider.ProviderName,
+			cloudRunner.Provider.RedactedToken,
+			cloudRunner.Provider.SSHKeyID,
+			cloudRunner.Provider.SnapshotID,
+		}
+		content = append(content, row)
+	}
+	table := tablewriter.NewWriter(os.Stderr)
+	table.SetAutoFormatHeaders(false)
+	table.SetHeader([]string{"Provider", "Token", "SSH Key ID", "Osmedeus Snapshot ID"})
+	table.SetBorders(tablewriter.Border{Left: true, Top: false, Right: true, Bottom: false})
+	table.SetCenterSeparator("|")
+	table.AppendBulk(content) // Add Bulk Data
+	table.Render()
+
+}

cmd/queue.go

@@ -0,0 +1,87 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/fatih/color"
+	"github.com/j3ssie/osmedeus/core"
+	"github.com/j3ssie/osmedeus/libs"
+	"github.com/j3ssie/osmedeus/utils"
+	jsoniter "github.com/json-iterator/go"
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	var queueCmd = &cobra.Command{
+		Use:     "queue",
+		Short:   "Running the scan with input from queue file",
+		Aliases: []string{"queq", "quee", "queu", "que"},
+		Long:    core.Banner(),
+		RunE:    runQueue,
+	}
+	queueCmd.PersistentFlags().StringVarP(&options.Queue.QueueFile, "queue-file", "Q", fmt.Sprintf("~/.%s/queue/queue-mimic.txt", libs.BINARY), "File contain list of target to simulate the queue")
+	queueCmd.PersistentFlags().BoolVar(&options.Queue.Add, "add", false, "Add new input to the queue file")
+	queueCmd.PersistentFlags().BoolVarP(&options.Queue.InputAsFile, "as-file", "F", false, "treat input as a file")
+	queueCmd.PersistentFlags().StringVar(&options.Queue.RawCommand, "cmd", "", "Raw Command to run")
+	queueCmd.SetHelpFunc(QueueHelp)
+	RootCmd.AddCommand(queueCmd)
+	queueCmd.PreRun = func(cmd *cobra.Command, args []string) {
+		if options.FullHelp {
+			cmd.Help()
+			os.Exit(0)
+		}
+	}
+}
+
+func runQueue(_ *cobra.Command, _ []string) error {
+	options.Queue.QueueFile = utils.NormalizePath(options.Queue.QueueFile)
+	options.Queue.QueueFolder = path.Dir(options.Queue.QueueFile)
+	if !utils.FolderExists(options.Queue.QueueFolder) {
+		utils.MakeDir(options.Queue.QueueFolder)
+	}
+
+	if options.Queue.Add {
+		addInput()
+		return nil
+	}
+
+	if !utils.FileExists(options.Queue.QueueFile) {
+		utils.WriteToFile(options.Queue.QueueFile, "")
+	}
+
+	content := utils.ReadingFileUnique(options.Queue.QueueFile)
+	if len(content) == 0 {
+		utils.WarnF("Queue file is empty: %v", options.Queue.QueueFile)
+		utils.WarnF("Consider to add a input to it:" + color.HiGreenString(" osmedeus queue --add -t example.com"))
+	} else {
+		utils.InforF("Queue file is not empty: %v", color.HiCyanString(options.Queue.QueueFile))
+		utils.InforF("Consider to delete it if you want a fresh scan")
+	}
+
+	core.QueueWatcher(options)
+	return nil
+}
+
+func addInput() {
+	utils.InforF("Adding new input to the queue file: %v", color.HiCyanString(options.Queue.QueueFile))
+	// osmedeus queue --add -t example.com
+	if options.Queue.RawCommand == "" {
+		utils.WriteToFile(options.Queue.QueueFile, strings.Join(options.Scan.Inputs, "\n"))
+		return
+	}
+
+	// osmedeus queue --add -t /tmp/cidr --cmd "osmedeus -t {{Input}} -m recon -w"
+	for _, target := range options.Scan.Inputs {
+		queueInput := libs.InputFormat{
+			Input:       target,
+			Command:     options.Queue.RawCommand,
+			InputAsFile: options.Queue.InputAsFile,
+		}
+		if line, ok := jsoniter.MarshalToString(queueInput); ok == nil {
+			utils.AppendToContent(options.Queue.QueueFile, line)
+		}
+	}
+}

cmd/report.go

@@ -0,0 +1,162 @@
+package cmd
+
+import (
+	"io"
+	"net/http"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"github.com/fatih/color"
+	"github.com/j3ssie/osmedeus/core"
+	"github.com/j3ssie/osmedeus/utils"
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	var reportCmd = &cobra.Command{
+		Use:   "report",
+		Short: "Show report of existing workspace",
+		Long:  core.Banner(),
+		RunE:  runReport,
+	}
+
+	var lsCmd = &cobra.Command{
+		Use:     "list",
+		Aliases: []string{"ls"},
+		Short:   "List all current existing workspace",
+		Long:    core.Banner(),
+		RunE:    runReportList,
+	}
+	reportCmd.AddCommand(lsCmd)
+
+	var viewCmd = &cobra.Command{
+		Use:     "view",
+		Aliases: []string{"vi", "v"},
+		Short:   "View all reports of existing workspace",
+		Long:    core.Banner(),
+		RunE:    runReportView,
+	}
+	reportCmd.AddCommand(viewCmd)
+
+	var extractCmd = &cobra.Command{
+		Use:     "extract",
+		Aliases: []string{"ext", "ex", "e"},
+		Short:   "Extract a compressed workspace",
+		Long:    core.Banner(),
+		RunE:    runReportExtract,
+	}
+	extractCmd.Flags().StringVar(&options.Report.ExtractFolder, "dest", "", "Destination folder to extract data to")
+	reportCmd.AddCommand(extractCmd)
+
+	var compressCmd = &cobra.Command{
+		Use:     "compress",
+		Aliases: []string{"com", "compr", "compres", "c"},
+		Short:   "Create a backup of the selected workspace",
+		Long:    core.Banner(),
+		RunE:    runReportCompress,
+	}
+	reportCmd.AddCommand(compressCmd)
+
+	reportCmd.PersistentFlags().BoolVar(&options.Report.Raw, "raw", false, "Show all the file in the workspace")
+	reportCmd.PersistentFlags().StringVar(&options.Report.PublicIP, "ip", "", "Show downloadable file with the given IP address")
+	reportCmd.PersistentFlags().BoolVar(&options.Report.Static, "static", false, "Show report file with Prefix Static")
+	reportCmd.SetHelpFunc(ReportHelp)
+	RootCmd.AddCommand(reportCmd)
+	reportCmd.PreRun = func(cmd *cobra.Command, args []string) {
+		if options.FullHelp {
+			cmd.Help()
+			os.Exit(0)
+		}
+	}
+}
+
+func runReportList(_ *cobra.Command, _ []string) error {
+	core.ListWorkspaces(options)
+	return nil
+}
+
+func runReportView(_ *cobra.Command, _ []string) error {
+	if options.Report.PublicIP == "" {
+		if utils.GetOSEnv("IPAddress", "127.0.0.1") == "127.0.0.1" {
+			options.Report.PublicIP = utils.GetOSEnv("IPAddress", "127.0.0.1")
+		}
+	}
+
+	if options.Report.PublicIP == "0" || options.Report.PublicIP == "0.0.0.0" {
+		options.Report.PublicIP = getPublicIP()
+	}
+
+	if len(options.Scan.Inputs) == 0 {
+		core.ListWorkspaces(options)
+		utils.InforF("Please select workspace to view report. Try %s", color.HiCyanString(`'osmedeus report view -t target.com'`))
+		return nil
+	}
+
+	for _, target := range options.Scan.Inputs {
+		core.ListSingleWorkspace(options, target)
+	}
+	return nil
+}
+
+func runReportExtract(_ *cobra.Command, _ []string) error {
+	var err error
+	if options.Report.ExtractFolder == "" {
+		options.Report.ExtractFolder = options.Env.WorkspacesFolder
+	} else {
+		options.Report.ExtractFolder, err = filepath.Abs(filepath.Dir(options.Report.ExtractFolder))
+		if err != nil {
+			return err
+		}
+	}
+
+	for _, input := range options.Scan.Inputs {
+		core.ExtractBackup(input, options)
+
+		target := strings.ReplaceAll(path.Base(input), ".tar.gz", "")
+		core.ListSingleWorkspace(options, target)
+	}
+
+	return nil
+}
+
+func runReportCompress(_ *cobra.Command, _ []string) error {
+	for _, target := range options.Scan.Inputs {
+		core.CompressWorkspace(target, options)
+	}
+	return nil
+}
+
+func runReport(_ *cobra.Command, args []string) error {
+	if options.Report.PublicIP == "" {
+		if utils.GetOSEnv("IPAddress", "127.0.0.1") == "127.0.0.1" {
+			options.Report.PublicIP = utils.GetOSEnv("IPAddress", "127.0.0.1")
+		}
+	}
+
+	if options.Report.PublicIP == "0" || options.Report.PublicIP == "0.0.0.0" {
+		options.Report.PublicIP = getPublicIP()
+	}
+
+	if len(args) == 0 {
+		core.ListWorkspaces(options)
+	}
+
+	return nil
+}
+
+func getPublicIP() string {
+	utils.DebugF("getting Public IP Address")
+	req, err := http.Get("https://api.ipify.org")
+	if err != nil {
+		return "127.0.0.1"
+	}
+	defer req.Body.Close()
+
+	body, err := io.ReadAll(req.Body)
+	if err != nil {
+		return "127.0.0.1"
+	}
+	return string(body)
+}

cmd/root.go

@@ -0,0 +1,147 @@
+package cmd
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/j3ssie/osmedeus/core"
+	"github.com/j3ssie/osmedeus/libs"
+	"github.com/j3ssie/osmedeus/utils"
+	"github.com/spf13/cobra"
+)
+
+var options = libs.Options{}
+var RootCmd = &cobra.Command{
+	Use:   libs.BINARY,
+	Short: fmt.Sprintf("%s - %s", libs.BINARY, libs.DESC),
+	Long:  core.Banner(),
+}
+
+// Execute main function
+func Execute() {
+	if err := RootCmd.Execute(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}
+
+func init() {
+	RootCmd.PersistentFlags().StringVar(&options.Env.RootFolder, "rootFolder", fmt.Sprintf("~/.%s/", libs.BINARY), "The main folder where all configurations are stored")
+	RootCmd.PersistentFlags().StringVar(&options.Env.BaseFolder, "baseFolder", fmt.Sprintf("~/%s-base/", libs.BINARY), "Base Folder which is store data, binaries and workflows")
+	RootCmd.PersistentFlags().StringVar(&options.Env.DataFolder, "dataFolder", fmt.Sprintf("~/%s-base/data", libs.BINARY), "Data folder which is store wordlists, payloads, etc")
+	RootCmd.PersistentFlags().StringVar(&options.Env.WorkFlowsFolder, "wfFolder", "", fmt.Sprintf("Custom Workflow folder (default will get from '$HOME/%s-base/workflow')", libs.BINARY))
+	RootCmd.PersistentFlags().StringVar(&options.ConfigFile, "configFile", fmt.Sprintf("~/.%s/config.yaml", libs.BINARY), "Main configurations file")
+
+	// Workspace folder
+	RootCmd.PersistentFlags().StringVarP(&options.Env.WorkspacesFolder, "wsFolder", "W", fmt.Sprintf("~/workspaces-%s", libs.BINARY), "The main data folder within the workspaces where all scan results are stored")
+
+	// parse target as global flag
+	RootCmd.PersistentFlags().StringSliceVarP(&options.Scan.Inputs, "target", "t", []string{}, "The target you want to run/execute")
+	RootCmd.PersistentFlags().StringVarP(&options.Scan.InputList, "targets", "T", "", "List of target as a file")
+
+	// Scan command
+	RootCmd.PersistentFlags().IntVarP(&options.Concurrency, "concurrency", "c", 1, "Concurrency level (recommend to keep it as 1 on machine has RAM smaller than 2GB)")
+	RootCmd.PersistentFlags().StringSliceVarP(&options.Scan.Modules, "module", "m", []string{}, "Target to running")
+	RootCmd.PersistentFlags().StringVarP(&options.Scan.Flow, "flow", "f", "general", "Flow name for running (default: general)")
+	RootCmd.PersistentFlags().StringVarP(&options.Scan.CustomWorkspace, "workspace", "w", "", "Name of workspace (default is same as target)")
+	RootCmd.PersistentFlags().StringSliceVarP(&options.Scan.Params, "params", "p", []string{}, "Custom params -p='foo=bar' (Multiple -p flags are accepted)")
+	RootCmd.PersistentFlags().StringVar(&options.Scan.SuffixName, "suffix", "", "Suffix string for file converted (default: randomly)")
+	RootCmd.PersistentFlags().IntVarP(&options.Threads, "threads-hold", "B", 0, "Threads hold for each module (default: number of CPUs)")
+	RootCmd.PersistentFlags().StringVar(&options.Tactics, "tactic", "default", "Choosing the tactic for running workflow from [default aggressive gently]")
+	RootCmd.PersistentFlags().StringVar(&options.Scan.ParamsFile, "params-file", "", "Custom file params --params-file=params.yaml")
+
+	// cloud flags
+	RootCmd.PersistentFlags().BoolVar(&options.Cloud.EnableChunk, "chunk", false, "Enable chunk mode")
+	RootCmd.PersistentFlags().BoolVar(&options.Scan.RemoteCall, "from-remote", false, "Invocation from a remote machine")
+	RootCmd.PersistentFlags().IntVarP(&options.Cloud.NumberOfParts, "chunk-parts", "P", 0, "Number of chunks file to split (default: equal with concurrency)")
+	RootCmd.PersistentFlags().StringVar(&options.Cloud.ChunkInputs, "chunkFolder", "/tmp/chunk-inputs/", "Temp Folder to store chunk inputs")
+	RootCmd.PersistentFlags().StringVar(&options.Timeout, "timeout", "", "Global timeout for each step (e.g: 60s, 30m, 2h)")
+	RootCmd.PersistentFlags().StringVar(&options.Cloud.Size, "size", "", "Override Size of cloud provider (default will get from 'cloud/provider.yaml')")
+	RootCmd.PersistentFlags().StringVar(&options.Cloud.Region, "region", "", "Override Region of cloud provider (default will get from 'cloud/provider.yaml')")
+	RootCmd.PersistentFlags().StringVar(&options.Cloud.Token, "token", "", "Override token of cloud provider (default will get from 'cloud/provider.yaml')")
+	RootCmd.PersistentFlags().StringVar(&options.Cloud.TokensFile, "token-file", "", "File contains list token of cloud providers")
+	RootCmd.PersistentFlags().StringVar(&options.Cloud.Provider, "provider", "", "Provider config file (default will get from 'cloud/provider.yaml')")
+	RootCmd.PersistentFlags().BoolVar(&options.Cloud.ReBuildBaseImage, "rebuild", false, "Forced to rebuild the images event though the version didn't change")
+
+	// mics option
+	RootCmd.PersistentFlags().StringVar(&options.LogFile, "log", "", fmt.Sprintf("Log File (default will store in '%s')", libs.LDIR))
+	RootCmd.PersistentFlags().StringVarP(&options.ScanID, "sid", "s", "", "Scan ID to continue the scan without create new scan record")
+	RootCmd.PersistentFlags().BoolVarP(&options.Resume, "resume", "R", false, "Enable Resume mode to skip modules that have already been finished")
+	RootCmd.PersistentFlags().BoolVar(&options.Debug, "debug", false, "Enable Debug output")
+	RootCmd.PersistentFlags().BoolVarP(&options.Quite, "quite", "q", false, "Show only essential information")
+	RootCmd.PersistentFlags().BoolVar(&options.FullHelp, "hh", false, "Show full help message")
+	RootCmd.PersistentFlags().BoolVar(&options.WildCardCheck, "ww", false, "Check for wildcard target")
+	RootCmd.PersistentFlags().BoolVar(&options.DisableValidateInput, "nv", false, "Disable Validate Input")
+	RootCmd.PersistentFlags().BoolVar(&options.Update.NoUpdate, "nu", false, "Disable Update options")
+	RootCmd.PersistentFlags().BoolVarP(&options.EnableFormatInput, "format-input", "J", false, "Enable special input format")
+	RootCmd.PersistentFlags().IntVar(&options.MDCodeBlockLimit, "block-size", 10000, "Size limit for code block tags to before it's automatic truncation")
+
+	// disable options
+	RootCmd.PersistentFlags().BoolVar(&options.NoNoti, "nn", false, "No notification")
+	RootCmd.PersistentFlags().BoolVar(&options.NoBanner, "nb", false, "No banner")
+	RootCmd.PersistentFlags().BoolVarP(&options.NoGit, "no-git", "N", false, "No git storage")
+	RootCmd.PersistentFlags().BoolVarP(&options.NoClean, "no-clean", "C", false, "No clean junk output")
+	RootCmd.PersistentFlags().BoolVar(&options.NoPreRun, "no-prerun", false, "Disable pre run scripts")
+	RootCmd.PersistentFlags().BoolVar(&options.NoPostRun, "no-postrun", false, "Disable post run scripts")
+	RootCmd.PersistentFlags().StringSliceVarP(&options.Exclude, "exclude", "x", []string{}, "Exclude module name (Multiple -x flags are accepted)")
+	RootCmd.PersistentFlags().BoolVarP(&options.CustomGit, "git", "g", false, "Use custom Git repo")
+
+	// sync options
+	RootCmd.PersistentFlags().BoolVar(&options.EnableDeStorage, "des", false, "Enable Dedicated Storages")
+	RootCmd.PersistentFlags().BoolVar(&options.GitSync, "sync", false, "Enable Sync Check before doing git push")
+	RootCmd.PersistentFlags().IntVar(&options.SyncTimes, "sync-timee", 15, "Number of times to check before force push")
+	RootCmd.PersistentFlags().IntVar(&options.PollingTime, "poll-timee", 100, "Number of seconds to sleep before do next sync check")
+	RootCmd.PersistentFlags().BoolVar(&options.NoCdn, "no-cdn", false, "Disable CDN feature")
+	RootCmd.PersistentFlags().BoolVarP(&options.EnableBackup, "backup", "b", false, "Backup the result after the scan is done")
+
+	// update options
+	RootCmd.PersistentFlags().BoolVar(&options.Update.IsUpdateBin, "bin", false, "Update binaries too")
+	RootCmd.PersistentFlags().BoolVar(&options.Update.EnableUpdate, "update", false, "Enable auto update")
+	RootCmd.PersistentFlags().StringVar(&options.Update.UpdateFolder, "update-folder", "/tmp/osm-update", "Folder to clone the update folder")
+
+	RootCmd.SetHelpFunc(RootHelp)
+	cobra.OnInitialize(initConfig)
+	RootCmd.PreRun = func(cmd *cobra.Command, args []string) {
+		if options.FullHelp {
+			cmd.Help()
+			os.Exit(0)
+		}
+	}
+}
+
+// initConfig reads in config file and ENV variables if set.
+func initConfig() {
+	if options.JsonOutput {
+		options.Quite = true
+	}
+
+	/* Really Start the program */
+	utils.InitLog(&options)
+	utils.InitHTTPClient()
+	if err := core.InitConfig(&options); err != nil {
+		utils.ErrorF("config file does not writable: %v", options.ConfigFile)
+		utils.BlockF("fatal", "Make sure you are login as 'root user' if your installation done via root user")
+	}
+	core.ParsingConfig(&options)
+
+	// parse inputs
+	if options.Scan.InputList != "" {
+		if utils.FileExists(options.Scan.InputList) {
+			options.Scan.Inputs = append(options.Scan.Inputs, utils.ReadingFileUnique(options.Scan.InputList)...)
+		}
+	}
+
+	// detect if anything came from stdin
+	stat, _ := os.Stdin.Stat()
+	if (stat.Mode() & os.ModeCharDevice) == 0 {
+		sc := bufio.NewScanner(os.Stdin)
+		for sc.Scan() {
+			target := strings.TrimSpace(sc.Text())
+			if err := sc.Err(); err == nil && target != "" {
+				options.Scan.Inputs = append(options.Scan.Inputs, target)
+			}
+		}
+	}
+}

cmd/scan.go

@@ -0,0 +1,80 @@
+package cmd
+
+import (
+	"os"
+	"strings"
+	"sync"
+
+	"github.com/fatih/color"
+	"github.com/j3ssie/osmedeus/core"
+	"github.com/j3ssie/osmedeus/libs"
+	"github.com/j3ssie/osmedeus/utils"
+	"github.com/panjf2000/ants"
+	"github.com/spf13/cobra"
+	"golang.org/x/text/cases"
+	"golang.org/x/text/language"
+)
+
+func init() {
+	var scanCmd = &cobra.Command{
+		Use:   "scan",
+		Short: "Conduct a scan following a predetermined flow/module",
+		Long:  core.Banner(),
+		RunE:  runScan,
+	}
+
+	scanCmd.SetHelpFunc(ScanHelp)
+	RootCmd.AddCommand(scanCmd)
+	scanCmd.PreRun = func(cmd *cobra.Command, args []string) {
+		if options.FullHelp {
+			cmd.Help()
+			os.Exit(0)
+		}
+	}
+}
+
+func runScan(_ *cobra.Command, _ []string) error {
+	utils.GoodF("%v %v by %v", cases.Title(language.Und, cases.NoLower).String(libs.BINARY), libs.VERSION, color.HiMagentaString(libs.AUTHOR))
+	utils.InforF("Storing the log file to: %v", color.CyanString(options.LogFile))
+
+	var wg sync.WaitGroup
+	p, _ := ants.NewPoolWithFunc(options.Concurrency, func(i interface{}) {
+		// really start to scan
+		CreateRunner(i)
+		wg.Done()
+	}, ants.WithPreAlloc(true))
+	defer p.Release()
+
+	if options.Cloud.EnableChunk {
+		for _, target := range options.Scan.Inputs {
+			chunkTargets := HandleChunksInputs(target)
+			for _, chunkTarget := range chunkTargets {
+				wg.Add(1)
+				_ = p.Invoke(chunkTarget)
+			}
+		}
+	} else {
+		for _, target := range options.Scan.Inputs {
+			wg.Add(1)
+			_ = p.Invoke(strings.TrimSpace(target))
+		}
+	}
+
+	wg.Wait()
+	return nil
+}
+
+func CreateRunner(j interface{}) {
+	target := j.(string)
+	if core.IsRootDomain(target) && options.Scan.Flow == "general" && len(options.Scan.Modules) == 0 {
+		utils.WarnF("looks like you scanning a subdomain '%s' with general flow. The result might be much less than usual", color.HiCyanString(target))
+		utils.WarnF("Better input should be root domain with TLD like '-t target.com'")
+	}
+
+	runner, err := core.InitRunner(target, options)
+	if err != nil {
+		utils.ErrorF("Error init runner with: %s", target)
+		return
+	}
+	runner.Start()
+}

cmd/server.go

@@ -0,0 +1,49 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/fatih/color"
+	"github.com/j3ssie/osmedeus/core"
+	"github.com/j3ssie/osmedeus/libs"
+	"github.com/j3ssie/osmedeus/server"
+	"github.com/j3ssie/osmedeus/utils"
+	"github.com/spf13/cobra"
+	"golang.org/x/text/cases"
+	"golang.org/x/text/language"
+)
+
+func init() {
+	var serverCmd = &cobra.Command{
+		Use:   "server",
+		Short: "Start Web Server",
+		Long:  core.Banner(),
+		RunE:  runServer,
+	}
+	serverCmd.Flags().String("host", "0.0.0.0", "IP address to bind the server")
+	serverCmd.Flags().String("port", "8000", "Port")
+	serverCmd.Flags().IntVar(&options.Server.PollingTime, "poll-time", 60, "Polling time to check next task")
+	serverCmd.Flags().BoolVar(&options.Server.DisableSSL, "disable-ssl", false, "Disable workspaces directory listing")
+	serverCmd.Flags().BoolVar(&options.Server.DisableWorkspaceListing, "disable-listing", false, "Disable workspaces directtory listing")
+	serverCmd.Flags().BoolVar(&options.Server.PreFork, "prefork", false, "Enable Prefork mode for the api server")
+	serverCmd.Flags().BoolVarP(&options.Server.NoAuthen, "no-auth", "A", false, "Disable authentication for the api server")
+
+	serverCmd.SetHelpFunc(ServerHelp)
+	RootCmd.AddCommand(serverCmd)
+	serverCmd.PreRun = func(cmd *cobra.Command, args []string) {
+		if options.FullHelp {
+			cmd.Help()
+			os.Exit(0)
+		}
+	}
+}
+
+func runServer(cmd *cobra.Command, _ []string) error {
+	host, _ := cmd.Flags().GetString("host")
+	port, _ := cmd.Flags().GetString("port")
+	options.Server.Bind = fmt.Sprintf("%v:%v", host, port)
+	utils.GoodF("%v %v by %v", cases.Title(language.Und, cases.NoLower).String(libs.BINARY), libs.VERSION, color.HiMagentaString(libs.AUTHOR))
+	server.StartServer(options)
+	return nil
+}

cmd/update.go

@@ -0,0 +1,77 @@
+package cmd
+
+import (
+	"os"
+
+	"github.com/j3ssie/osmedeus/core"
+	"github.com/j3ssie/osmedeus/utils"
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	var updateCmd = &cobra.Command{
+		Use:   "update",
+		Short: "Check latest Update",
+		Long:  core.Banner(),
+		RunE:  runUpdate,
+	}
+	updateCmd.Flags().String("meta", "", "Custom MetaData URL")
+	updateCmd.Flags().Bool("F", false, "Shortcut for force update and clean old data at the same time")
+	updateCmd.Flags().BoolVar(&options.Update.ForceUpdate, "force", false, "Force update")
+	updateCmd.Flags().BoolVar(&options.Update.CleanOldData, "clean", false, "Clean up old Data")
+	updateCmd.Flags().BoolVar(&options.Update.VulnUpdate, "vuln", false, "Update Vulnerability Database only")
+	// generate update meta data
+	updateCmd.Flags().StringVar(&options.Update.UpdateURL, "update-url", "", "The script URL to download update")
+	updateCmd.Flags().StringVar(&options.Update.GenerateMeta, "gen", "", "Generate metadata for update")
+	RootCmd.AddCommand(updateCmd)
+	updateCmd.PreRun = func(cmd *cobra.Command, args []string) {
+		if options.FullHelp {
+			cmd.Help()
+			os.Exit(0)
+		}
+	}
+}
+
+func runUpdate(cmd *cobra.Command, _ []string) error {
+	meta, _ := cmd.Flags().GetString("meta")
+	if meta != "" {
+		options.Update.MetaDataURL = meta
+	}
+
+	forcedUpdateandClean, _ := cmd.Flags().GetBool("F")
+	if forcedUpdateandClean {
+		options.Update.ForceUpdate = true
+		options.Update.CleanOldData = true
+	}
+
+	if options.Update.GenerateMeta != "" {
+		core.GenerateMetaData(options)
+		return nil
+	}
+
+	if options.Update.VulnUpdate {
+		core.UpdateVuln(options)
+		return nil
+	}
+
+	var shouldUpdate bool
+	if options.Update.UpdateURL == "" {
+		options.Update.UpdateURL = core.GetUpdateURL(options)
+	}
+
+	if options.Update.ForceUpdate {
+		shouldUpdate = true
+		utils.InforF("Force to Update latest release")
+	} else {
+		shouldUpdate = core.CheckUpdate(&options)
+	}
+
+	if shouldUpdate {
+		err := core.RunUpdate(options)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

cmd/usage.go

@@ -0,0 +1,313 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/fatih/color"
+	"github.com/j3ssie/osmedeus/core"
+	"github.com/j3ssie/osmedeus/libs"
+	"github.com/spf13/cobra"
+)
+
+// RootUsage base help
+func RootUsage() {
+	var h string
+	h += ScanUsage()
+	h += CloudUsage()
+	h += QueueUsage()
+	h += ReportUsage()
+	h += UtilsUsage()
+	fmt.Println(h)
+}
+
+func ScanExmaples() string {
+	h := color.HiCyanString("Example Scan Commands:")
+	h += color.HiBlueString("\n  ## Start a simple scan with default 'general' flow\n")
+	h += "  osmedeus scan -t sample.com\n"
+
+	h += color.HiBlueString("\n  ## Start a general scan but exclude some of the module\n")
+	h += "  osmedeus scan -t sample.com -x screenshot -x spider\n"
+
+	h += color.HiBlueString("\n  ## Start a scan directly with a module with inputs as a list of http domains like this https://sub.example.com\n")
+	h += "  osmedeus scan -m content-discovery -t http-file.txt\n"
+
+	h += color.HiBlueString("\n  ## Initiate the scan using a speed option other than the default setting\n")
+	h += "  osmedeus scan -f vuln --tactic gently -t sample.com\n"
+	h += "  osmedeus scan --threads-hold=10 -t sample.com\n"
+	h += "  osmedeus scan -B 5 -t sample.com\n"
+
+	h += color.HiBlueString("\n  ## Start a simple scan with other flow\n")
+	h += "  osmedeus scan -f vuln -t sample.com\n"
+	h += "  osmedeus scan -f extensive -t sample.com -t another.com\n"
+	h += "  osmedeus scan -f urls -t list-of-urls.txt\n"
+
+	h += color.HiBlueString("\n  ## Scan list of targets\n")
+	h += "  osmedeus scan -T list_of_targets.txt\n"
+	h += "  osmedeus scan -f vuln -T list-of-targets.txt\n"
+
+	h += color.HiBlueString("\n  ## Performing static vulnerability scan and secret scan on a git repo\n")
+	h += "  osmedeus scan -m repo-scan -t https://github.com/j3ssie/sample-repo\n"
+	h += "  osmedeus scan -m repo-scan -t /tmp/source-code-folder\n"
+	h += "  osmedeus scan -m repo-scan -T list-of-repo.txt\n"
+
+	h += color.HiBlueString("\n  ## Scan for CIDR with file contains CIDR with the format '1.2.3.4/24'\n")
+	h += "  osmedeus scan -f cidr -t list-of-ciders.txt\n"
+	h += "  osmedeus scan -f cidr -t '1.2.3.4/24' # this will auto convert the single input to the file and run\n"
+
+	h += color.HiBlueString("\n  ## Directly run on vuln scan and directory scan on list of domains\n")
+	h += "  osmedeus scan -f domains -t list-of-domains.txt\n"
+	h += "  osmedeus scan -f vuln-and-dirb -t list-of-domains.txt\n"
+
+	h += color.HiBlueString("\n  ## Use a custom wordlist\n")
+	h += "  osmedeus scan -t sample.com -p 'wordlists={{Data}}/wordlists/content/big.txt'\n"
+
+	h += color.HiBlueString("\n  ## Use a custom wordlist\n")
+	h += "  cat list_of_targets.txt | osmedeus scan -c 2\n"
+
+	h += color.HiBlueString("\n  ## Start a normal scan and backup entire workflow folder to the backup folder\n")
+	h += "  osmedeus scan --backup -f domains -t list-of-subdomains.txt\n"
+
+	h += color.HiBlueString("\n  ## Start the scan with chunk inputs to review the output way more much faster\n")
+	h += "  osmedeus scan --chunk --chunk-parts 20 -f cidr -t list-of-100-cidr.txt\n"
+
+	h += color.HiBlueString("\n  ## Continuously run the scan on a target right after it finished\n")
+	h += "  osmedeus utils cron --for --cmd 'osmedeus scan -t example.com'\n"
+
+	h += color.HiBlueString("\n  ## Backing up all workspaces\n")
+	h += "  ls ~/workspaces-osmedeus | osmedeus report compress\n"
+
+	h += "\n"
+	return h
+}
+
+func ScanUsage() string {
+	h := ScanExmaples()
+	h += color.HiCyanString("\nScan Usage:\n")
+	h += "  osmedeus scan -f [flowName] -t [target] \n"
+	h += "  osmedeus scan -m [modulePath] -T [targetsFile] \n"
+	h += "  osmedeus scan -f /path/to/flow.yaml -t [target] \n"
+	h += "  osmedeus scan -m /path/to/module.yaml -t [target] --params 'port=9200'\n"
+	h += "  osmedeus scan -m /path/to/module.yaml -t [target] -l /tmp/log.log\n"
+	h += "  osmedeus scan --tactic aggressive -m module -t [target] \n"
+	h += "  cat targets | osmedeus scan -f sample\n"
+
+	h += color.HiCyanString("\nPractical Scan Usage:\n")
+	h += "  osmedeus scan -T list_of_targets.txt -W custom_workspaces\n"
+	h += "  osmedeus scan -t target.com -w workspace_name --debug\n"
+	h += "  osmedeus scan -f general -t sample.com\n"
+	h += "  osmedeus scan --tactic aggressive -f general -t sample.com\n"
+	h += "  osmedeus scan -f extensive -t sample.com -t another.com\n"
+	h += "  cat list_of_urls.txt | osmedeus scan -f urls\n"
+	h += "  osmedeus scan --threads-hold=15 -f cidr -t 1.2.3.4/24\n"
+	h += "  osmedeus scan -m ~/.osmedeus/core/workflow/test/dirbscan.yaml -t list_of_urls.txt\n"
+	h += "  osmedeus scan --wfFolder ~/custom-workflow/ -f your-custom-workflow -t list_of_urls.txt\n"
+	h += "  osmedeus scan --chunk --chunk-part 40 -c 2 -f cidr -t list-of-cidr.txt\n"
+	return h
+}
+
+func UtilsUsage() string {
+	h := color.HiCyanString("\nUtilities Usage:\n")
+	h += color.HiBlueString("  ## Health Utility\n")
+	h += "  osmedeus health \n"
+	h += "  osmedeus health git\n"
+	h += "  osmedeus health cloud\n"
+	h += "  osmedeus version --json \n"
+	h += "\n"
+
+	h += color.HiBlueString("  ## Set the base threads hold\n")
+	h += "  osmedeus config set --threads-hold=10\n"
+	h += "\n"
+
+	h += color.HiBlueString("  ## Update utilities\n")
+	h += "  osmedeus update \n"
+	h += "  osmedeus update --vuln\n"
+	h += "  osmedeus update --force --clean \n"
+	h += "\n"
+
+	h += color.HiBlueString("  ## Workflow utilities\n")
+	h += "  osmedeus workflow list \n"
+	h += "  osmedeus workflow view -f general\n"
+	h += "\n"
+
+	h += color.HiBlueString("  ## Tmux utilities\n")
+	h += "  osmedeus utils tmux ls \n"
+	h += "  osmedeus utils tmux logs -A -l 10 \n"
+	h += "\n"
+
+	h += color.HiBlueString("  ## Process utilities\n")
+	h += "  osmedeus utils ps \n"
+	h += "  osmedeus utils ps --proc 'jaeles' \n"
+	h += "\n"
+
+	h += color.HiBlueString("  ## List all the sub proccess running by osmedeus\n")
+	h += "  osmedeus utils ps --osm \n"
+	h += "\n"
+
+	h += color.HiBlueString("  ## Kill all the sub proccess running by osmedeus\n")
+	h += "  osmedeus utils ps --osm --kill \n"
+	h += "\n"
+
+	h += color.HiBlueString("  ## Cron utilities\n")
+	h += "  osmedeus utils cron --cmd 'osmdeus scan -t example.com' --sch 60\n"
+	h += "  osmedeus utils cron --for --cmd 'osmedeus scan -t example.com'\n"
+
+	return h
+}
+
+func ConfigUsage() string {
+	h := color.HiCyanString("\nConfig Usage:\n")
+	h += "  osmedeus config [action] [OPTIONS] \n"
+	h += "  osmedeus config init -p https://github.com/j3ssie/osmedeus-plugins\n"
+	h += "  osmedeus config --user newusser --pass newpassword\n"
+	h += "  osmedeus config clean \n"
+	h += "  osmedeus config delete -t woskapce \n"
+	h += "  osmedeus config delete -w workspace_name \n"
+	h += "  osmedeus config set --threads-hold=10 \n"
+	return h
+}
+
+func QueueUsage() string {
+	h := color.HiCyanString("\nQueue Usage:\n")
+	h += "  osmedeus queue -Q /tmp/queue-file.txt -c 2\n"
+	h += "  osmedeus queue --add -t example.com -Q /tmp/queue-file.txt \n"
+	return h
+}
+
+func CloudUsage() string {
+	h := color.HiCyanString("\nProvider Usage:\n")
+	h += "  osmedeus provider wizard \n"
+	h += "  osmedeus provider validate \n"
+	h += "  osmedeus provider build --token xxx --rebuild --ic\n"
+
+	h += "  osmedeus provider health --debug \n"
+	h += "  osmedeus provider health --for \n"
+	h += "  osmedeus provider create --name 'sample' \n"
+	h += "  osmedeus provider delete --id 34317111 --id 34317112 \n"
+	h += "  osmedeus provider list \n"
+
+	h += color.HiCyanString("\nCloud Usage:\n")
+	h += "  osmedeus cloud -f [flowName] -t [target] \n"
+	h += "  osmedeus cloud -f [flowName] -T [targetFile] --no-del\n"
+	h += "  osmedeus cloud -m [modulePath] -t [target] \n"
+	h += "  osmedeus cloud -c 5 -f [flowName] -T [targetsFile] \n"
+	h += "  osmedeus cloud --token xxx -c 5 -f [flowName] -T [targetsFile] \n"
+	h += "  osmedeus cloud --chunk -c 5 -f [flowName] -t [targetsFile] \n"
+
+	return h
+}
+
+func ReportUsage() string {
+	h := color.HiCyanString("\nReport Usage:\n")
+	h += "  osmedeus report list\n"
+	h += "  osmedeus report extract -t target.com.tar.gz\n"
+	h += "  osmedeus report extract -t target.com.tar.gz --dest .\n"
+	h += "  osmedeus report compress -t target.com\n"
+	h += "  osmedeus report view --raw -t target.com\n"
+	h += "  osmedeus report view --static -t target.com\n"
+	h += "  osmedeus report view --static --ip 0 -t target.com\n"
+	return h
+}
+
+func ServerUsage() string {
+	h := color.HiCyanString("\nServer Usage:\n")
+	h += "  osmedeus server --port 5000\n"
+	h += "  osmedeus server --disable-ssl\n"
+	h += "  osmedeus server -A --disable-ssl\n"
+	return h
+}
+
+func QueueHelp(cmd *cobra.Command, _ []string) {
+	fmt.Println(core.Banner())
+	fmt.Println(cmd.UsageString())
+	h := QueueUsage()
+	fmt.Println(h)
+	printDocs(cmd)
+}
+
+// ScanHelp scan help message
+func ScanHelp(cmd *cobra.Command, _ []string) {
+	fmt.Println(core.Banner())
+	if options.FullHelp {
+		fmt.Println(cmd.UsageString())
+	}
+	h := ScanUsage()
+	fmt.Println(h)
+	printDocs(cmd)
+}
+
+// CloudHelp scan help message
+func CloudHelp(cmd *cobra.Command, _ []string) {
+	fmt.Println(core.Banner())
+	if options.FullHelp {
+		fmt.Println(cmd.UsageString())
+	}
+	h := CloudUsage()
+	fmt.Println(h)
+	printDocs(cmd)
+}
+
+// ServerHelp scan help message
+func ServerHelp(cmd *cobra.Command, _ []string) {
+	fmt.Println(core.Banner())
+	if options.FullHelp {
+		fmt.Println(cmd.UsageString())
+	}
+	h := ServerUsage()
+	fmt.Println(h)
+	printDocs(cmd)
+}
+
+// ConfigHelp config help message
+func ConfigHelp(cmd *cobra.Command, _ []string) {
+	fmt.Println(core.Banner())
+	if options.FullHelp {
+		fmt.Println(cmd.UsageString())
+	}
+	h := ConfigUsage()
+
+	fmt.Println(h)
+	printDocs(cmd)
+}
+
+// UtilsHelp utils help message
+func UtilsHelp(cmd *cobra.Command, _ []string) {
+	fmt.Println(core.Banner())
+	if options.FullHelp {
+		fmt.Println(cmd.UsageString())
+	}
+	h := UtilsUsage()
+	fmt.Println(h)
+	printDocs(cmd)
+}
+
+// ReportHelp utils help message
+func ReportHelp(cmd *cobra.Command, _ []string) {
+	fmt.Println(core.Banner())
+	if options.FullHelp {
+		fmt.Println(cmd.UsageString())
+	}
+	h := ReportUsage()
+	fmt.Println(h)
+	printDocs(cmd)
+}
+
+// RootHelp print help message
+func RootHelp(cmd *cobra.Command, _ []string) {
+	fmt.Println(core.Banner())
+	if options.FullHelp {
+		fmt.Println(cmd.UsageString())
+	}
+	RootUsage()
+	printDocs(cmd)
+}
+
+func printDocs(cmd *cobra.Command) {
+	if !options.FullHelp {
+		if cmd.Use == libs.BINARY {
+			fmt.Printf("💡 For full help message, please run: %s\n", color.GreenString("osmedeus --hh"))
+		} else {
+			fmt.Printf("💡 For full help message, please run: %s or %s\n", color.GreenString("osmedeus --hh"), color.GreenString("osmedeus "+cmd.Use+" --hh"))
+		}
+	}
+	fmt.Printf("📖 Documentation can be found here: %s\n", color.GreenString(libs.DOCS))
+}

cmd/utils.go

@@ -0,0 +1,139 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/fatih/color"
+	"github.com/j3ssie/osmedeus/core"
+	"github.com/j3ssie/osmedeus/execution"
+	"github.com/j3ssie/osmedeus/libs"
+	"github.com/j3ssie/osmedeus/utils"
+	jsoniter "github.com/json-iterator/go"
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	var utilsCmd = &cobra.Command{
+		Use:     "utils",
+		Aliases: []string{"u", "util"},
+		Short:   "Utils to get some information from the system",
+		Long:    core.Banner(),
+		RunE:    runUtils,
+	}
+
+	var psCmd = &cobra.Command{
+		Use:   "ps",
+		Short: "Utility to get information about running process",
+		Long:  core.Banner(),
+		RunE:  runPs,
+	}
+	psCmd.Flags().StringSlice("proc", []string{}, "Process name")
+	psCmd.Flags().Bool("kill", false, "Kill the all processes")
+	psCmd.Flags().Bool("osm", false, "Osmedeus related process only")
+
+	var tmuxCmd = &cobra.Command{
+		Use:   "tmux",
+		Short: "Utility to get info from tmux",
+		Long:  core.Banner(),
+		RunE:  runTmux,
+	}
+
+	tmuxCmd.Flags().BoolVarP(&options.Tmux.ApplyAll, "all", "A", false, "Apply for all tmux sessions")
+	tmuxCmd.Flags().StringVarP(&options.Tmux.SelectedWindow, "name", "n", "", "Apply for all tmux sessions")
+	tmuxCmd.Flags().StringVarP(&options.Tmux.Exclude, "exclude", "e", "server", "Exclude tmux session")
+	tmuxCmd.Flags().IntVarP(&options.Tmux.Limit, "limit", "l", 0, "Size of output content")
+
+	var cronCmd = &cobra.Command{
+		Use:   "cron",
+		Short: "Utility to run command schedule",
+		Long:  core.Banner(),
+		RunE:  runCron,
+	}
+	cronCmd.Flags().IntVar(&options.Cron.Schedule, "sch", 0, "Number of minutes to schedule the job")
+	cronCmd.Flags().BoolVar(&options.Cron.Forever, "for", false, "Keep running forever right after the command done")
+	cronCmd.Flags().StringVar(&options.Cron.Command, "cmd", "", "Command to run")
+
+	// add command
+	utilsCmd.PersistentFlags().BoolVar(&options.JsonOutput, "json", false, "Output as JSON")
+	utilsCmd.AddCommand(cronCmd)
+	utilsCmd.AddCommand(tmuxCmd)
+	utilsCmd.AddCommand(psCmd)
+	utilsCmd.SetHelpFunc(UtilsHelp)
+	RootCmd.AddCommand(utilsCmd)
+
+	utilsCmd.PreRun = func(cmd *cobra.Command, args []string) {
+		if options.FullHelp {
+			cmd.Help()
+			os.Exit(0)
+		}
+	}
+}
+
+func runUtils(_ *cobra.Command, _ []string) error {
+	fmt.Println(UtilsUsage())
+	return nil
+}
+
+func runPs(cmd *cobra.Command, _ []string) error {
+	processes, _ := cmd.Flags().GetStringSlice("process")
+	osmRelated, _ := cmd.Flags().GetBool("osm")
+	killProcess, _ := cmd.Flags().GetBool("kill")
+
+	if osmRelated {
+		pids := execution.ListAllOsmedeusProcess()
+		for _, pid := range pids {
+			if killProcess {
+				utils.RunOSCommand(fmt.Sprintf("kill -9 %v", pid))
+			}
+		}
+		return nil
+	}
+
+	if len(processes) == 0 {
+		processes = append(processes, libs.BINARY)
+	}
+
+	for _, process := range processes {
+		pss := execution.GetOsmProcess(process)
+		for _, ps := range pss {
+			if options.JsonOutput {
+				if data, err := jsoniter.MarshalToString(ps); err == nil {
+					fmt.Println(data)
+				}
+				continue
+			}
+			fmt.Printf("pid:%v %s %v\n", color.HiCyanString("%v", ps.PID), color.HiMagentaString("--"), ps.Command)
+		}
+	}
+
+	return nil
+}
+
+func runTmux(_ *cobra.Command, args []string) error {
+	tmux, err := core.InitTmux(options)
+	if err != nil {
+		return err
+	}
+
+	for _, argument := range args {
+		switch argument {
+		case "l", "ls", "list":
+			tmux.ListTmux()
+		case "t", "log", "logs", "tai", "tail":
+			tmux.CatchSession()
+		}
+	}
+	return nil
+}
+
+func runCron(_ *cobra.Command, _ []string) error {
+	if options.Cron.Schedule == 0 && options.Cron.Forever == false {
+		return fmt.Errorf("missing '--sche' flag")
+	}
+	if options.Cron.Forever {
+		options.Cron.Schedule = -1
+	}
+	core.RunCron(options.Cron.Command, options.Cron.Schedule)
+	return nil
+}

cmd/version.go

@@ -0,0 +1,176 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"runtime"
+	"strings"
+	"time"
+
+	"github.com/j3ssie/osmedeus/core"
+	"github.com/j3ssie/osmedeus/libs"
+	"github.com/j3ssie/osmedeus/utils"
+	jsoniter "github.com/json-iterator/go"
+
+	//"github.com/mackerelio/go-osstat/cpu"
+	//"github.com/mackerelio/go-osstat/memory"
+	"github.com/spf13/cobra"
+	//"github.com/shirou/gopsutil/cpu"
+	"github.com/shirou/gopsutil/mem"
+)
+
+func init() {
+	var versionCmd = &cobra.Command{
+		Use:   "version",
+		Short: "Show core version",
+		Long:  core.Banner(),
+		RunE:  runVersion,
+	}
+	versionCmd.Flags().BoolVarP(&options.Verbose, "verbose", "V", false, "Show stat info too")
+	versionCmd.Flags().BoolVar(&options.JsonOutput, "json", false, "Output as JSON")
+	RootCmd.AddCommand(versionCmd)
+}
+
+func runVersion(_ *cobra.Command, _ []string) error {
+	if options.JsonOutput {
+		fmt.Println(PrintStat())
+		return nil
+	}
+
+	if !options.Verbose {
+		fmt.Printf("osmedeus %s by %s\n", libs.VERSION, libs.AUTHOR)
+	} else {
+		statInfo := PrintStat()
+		fmt.Printf("osmedeus %s by %s -- %s\n", libs.VERSION, libs.AUTHOR, statInfo)
+	}
+	return nil
+}
+
+// StatData overview struct
+type StatData struct {
+	CPU     string `json:"cpu"`
+	Mem     string `json:"mem"`
+	Name    string `json:"name"`
+	Version string `json:"version"`
+}
+
+// PrintStat print status
+func PrintStat() string {
+	data := GetStat()
+	if data.CPU.Idle == 0.0 {
+		return strings.TrimSpace(utils.Emojif(":thought_balloon:", "not responding"))
+	}
+	var cpu string
+	cpuUsage := 100.0 - data.CPU.Idle
+	if cpuUsage <= 20.0 {
+		cpu = utils.Emojif(":green_circle:", "  cpu: %0.2f", cpuUsage)
+	} else if (cpuUsage > 20.0) && (cpuUsage <= 50.0) {
+		cpu = utils.Emojif(":green_circle:", "  cpu: %0.2f", cpuUsage)
+	} else if (cpuUsage > 50.0) && (cpuUsage <= 80.0) {
+		cpu = utils.Emojif(":orange_circle:", "  cpu: %0.2f", cpuUsage)
+	} else {
+		cpu = utils.Emojif(":red_circle:", "  cpu: %0.2f", cpuUsage)
+	}
+
+	var mem string
+	memUsage := 100.0 - (data.Mem.Free/data.Mem.Total)*100
+	if memUsage <= 20.0 {
+		mem = utils.Emojif(":green_circle:", "  mem: %0.2f", memUsage)
+	} else if (memUsage > 20.0) && (memUsage <= 50.0) {
+		mem = utils.Emojif(":green_circle:", "  mem: %0.2f", memUsage)
+	} else if (memUsage > 50.0) && (memUsage <= 80.0) {
+		mem = utils.Emojif(":orange_circle:", "  mem: %0.2f", memUsage)
+	} else {
+		mem = utils.Emojif(":red_circle:", "  mem: %0.2f", memUsage)
+	}
+
+	name, _ := os.Hostname()
+	if options.JsonOutput {
+		stat := StatData{
+			CPU:     fmt.Sprintf("%v", cpuUsage),
+			Mem:     fmt.Sprintf("%v", memUsage),
+			Name:    name,
+			Version: fmt.Sprintf("osmedeus %s by %s", libs.VERSION, libs.AUTHOR),
+		}
+		if data, err := jsoniter.MarshalToString(stat); err == nil {
+			return data
+		}
+	}
+
+	return fmt.Sprintf("%s: %12s - %s", name, strings.TrimSpace(cpu), strings.TrimSpace(mem))
+}
+
+type ServerStatData struct {
+	CPU struct {
+		System float64
+		User   float64
+		Idle   float64
+	}
+	Mem struct {
+		Total  float64
+		Used   float64
+		Free   float64
+		Cached float64
+	}
+}
+
+// GetStat get stat data
+// func GetStat() ServerStatData {
+// 	var stat ServerStatData
+
+// 	before, err := cpu.Get()
+// 	if err != nil {
+// 		return stat
+// 	}
+// 	time.Sleep(time.Duration(1) * time.Second)
+// 	after, err := cpu.Get()
+// 	if err != nil {
+// 		return stat
+// 	}
+// 	total := float64(after.Total - before.Total)
+// 	stat.CPU.User = float64(after.User-before.User) / total * 100
+// 	stat.CPU.System = float64(after.System-before.System) / total * 100
+// 	stat.CPU.Idle = float64(after.Idle-before.Idle) / total * 100
+// 	// memory part
+// 	memory, err := memory.Get()
+// 	if err != nil {
+// 		return stat
+// 	}
+// 	stat.Mem.Total = float64(memory.Total+memory.SwapTotal) / (1024 * 1024 * 1024)
+// 	stat.Mem.Used = float64(memory.Used+memory.SwapUsed) / (1024 * 1024 * 1024)
+// 	stat.Mem.Used = float64(memory.Used+memory.SwapUsed) / (1024 * 1024 * 1024)
+// 	stat.Mem.Cached = float64(memory.Cached) / (1024 * 1024 * 1024)
+// 	stat.Mem.Free = float64(memory.Free+memory.SwapFree) / (1024 * 1024 * 1024)
+// 	return stat
+// }
+
+// GetStat gets stat data
+func GetStat() ServerStatData {
+	var stat ServerStatData
+
+	var before runtime.MemStats
+	runtime.ReadMemStats(&before)
+
+	time.Sleep(time.Duration(1) * time.Second)
+
+	var after runtime.MemStats
+	runtime.ReadMemStats(&after)
+
+	totalAlloc := float64(after.TotalAlloc - before.TotalAlloc)
+	sys := float64(after.Sys - before.Sys)
+	stat.CPU.User = totalAlloc / sys * 100
+	stat.CPU.System = sys / sys * 100
+	stat.CPU.Idle = 100 - stat.CPU.User - stat.CPU.System
+
+	// Memory part
+	memory, err := mem.VirtualMemory()
+	if err != nil {
+		return stat
+	}
+	stat.Mem.Total = float64(memory.Total) / (1024 * 1024 * 1024)
+	stat.Mem.Used = float64(memory.Used) / (1024 * 1024 * 1024)
+	stat.Mem.Cached = float64(memory.Cached) / (1024 * 1024 * 1024)
+	stat.Mem.Free = float64(memory.Free) / (1024 * 1024 * 1024)
+
+	return stat
+}

cmd/workflow.go

@@ -0,0 +1,200 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/fatih/color"
+	"github.com/j3ssie/osmedeus/core"
+	"github.com/j3ssie/osmedeus/utils"
+	"github.com/olekukonko/tablewriter"
+	"github.com/spf13/cobra"
+)
+
+func init() {
+
+	var workflowCmd = &cobra.Command{
+		Use:     "workflow",
+		Aliases: []string{"wf", "wl", "workflows", "wfs", "work", "works"},
+		Short:   "Listing all available workflows",
+		Long:    core.Banner(),
+	}
+
+	var workflowListCmd = &cobra.Command{
+		Use:     "list",
+		Aliases: []string{"ls"},
+		Short:   "Listing all available workflows",
+		Long:    core.Banner(),
+		RunE:    runWorkflow,
+	}
+
+	var workflowViewCmd = &cobra.Command{
+		Use:     "view",
+		Aliases: []string{"viwe", "ve", "vi", "v"},
+		Short:   "View details of a workflow",
+		Long:    core.Banner(),
+		RunE:    runWorkflowView,
+	}
+	workflowViewCmd.Flags().Bool("all", false, "View all of the workflows")
+
+	workflowCmd.AddCommand(workflowViewCmd)
+	workflowCmd.AddCommand(workflowListCmd)
+	workflowCmd.SetHelpFunc(UtilsHelp)
+	RootCmd.AddCommand(workflowCmd)
+
+	workflowCmd.PreRun = func(cmd *cobra.Command, args []string) {
+		if options.FullHelp {
+			cmd.Help()
+			os.Exit(0)
+		}
+	}
+}
+
+func runWorkflow(cmd *cobra.Command, _ []string) error {
+	listFlows()
+	fmt.Printf("\n------------------------------------------------------------\n")
+	listDefaultModules()
+	fmt.Printf("💡 For full help message, please run: %s or %s\n", color.GreenString("osmedeus --hh"), color.GreenString("osmedeus scan --hh"))
+	return nil
+}
+
+func runWorkflowView(cmd *cobra.Command, _ []string) error {
+	allFlows := core.ListFlow(options)
+	viewAll, _ := cmd.Flags().GetBool("all")
+
+	if viewAll {
+		for _, flow := range allFlows {
+			err := viewWorkflow(flow)
+			if err != nil {
+				utils.ErrorF("Error viewing workflow: %v", err)
+			}
+			fmt.Printf("\n------------------------------------------------------------\n\n")
+		}
+	} else {
+		err := viewWorkflow(options.Scan.Flow)
+		if err != nil {
+			utils.ErrorF("Error viewing workflow: %v", err)
+		}
+	}
+
+	h := color.HiCyanString("\n📄 Sample Usage:\n")
+	h += color.HiGreenString(" osmedeus scan -f %v", color.HiMagentaString(options.Scan.Flow)) + color.HiGreenString(" -t ") + color.HiMagentaString("[target]") + "\n"
+	h += color.HiGreenString(" osmedeus scan -f %v", color.HiMagentaString(options.Scan.Flow)) + color.HiGreenString(" -t ") + color.HiMagentaString("[target]") + color.HiGreenString(" -p ") + color.HiMagentaString("'enableSomething=false'") + "\n\n"
+	fmt.Printf(h)
+
+	fmt.Printf("💡 To list all of the workflows available, please run: %s\n", color.GreenString("osmedeus workflow ls"))
+	fmt.Printf("💡 For full help message, please run: %s or %s\n", color.GreenString("osmedeus --hh"), color.GreenString("osmedeus scan --hh"))
+	return nil
+}
+
+func viewWorkflow(workflowName string) error {
+	fmt.Printf("📖 Viewing workflow detail: %v\n\n", color.GreenString(workflowName))
+	allFlows := core.ListFlow(options)
+	flows := core.SelectFlow(workflowName, options)
+	if len(flows) == 0 {
+		utils.ErrorF("Flow not found in any of existing workflow [%v]", color.HiYellowString(strings.Join(allFlows, ", ")))
+		return fmt.Errorf("Flow %s not found", workflowName)
+	}
+	selectedWorkflow := flows[0]
+
+	var content [][]string
+	parsedFlow, err := core.ParseFlow(selectedWorkflow)
+	if err != nil {
+		utils.ErrorF("Error parsing flow: %v", selectedWorkflow)
+		return err
+	}
+
+	var totalSteps, totalModules int
+	parameters := make(map[string]string)
+	for _, param := range parsedFlow.Params {
+		for k, v := range param {
+			parameters[k] = v
+		}
+	}
+
+	for _, routine := range parsedFlow.Routines {
+		// select module depend on the flow type
+		if routine.FlowFolder != "" {
+			parsedFlow.Type = routine.FlowFolder
+		} else {
+			parsedFlow.Type = parsedFlow.DefaultType
+		}
+
+		modules := core.SelectModules(routine.Modules, options)
+
+		// loop through all modules to get the parameters
+		for _, module := range modules {
+			parsedModule, err := core.ParseModules(module)
+			if err != nil || parsedModule.Name == "" {
+				continue
+			}
+			for _, param := range parsedModule.Params {
+				for k, v := range param {
+
+					_, exist := parameters[k]
+					if parsedFlow.ForceParams && exist {
+						utils.DebugF("Skip override param: %v --> %v", k, v)
+						continue
+					}
+					parameters[k] = v
+				}
+
+			}
+			totalSteps += len(parsedModule.Steps)
+			totalModules++
+		}
+	}
+
+	var toggleFlags, skippingFlags []string
+	for key, value := range parameters {
+		if value == "true" {
+			value = color.GreenString(value)
+		} else if value == "false" {
+			value = color.RedString(value)
+		} else {
+
+			value = color.CyanString(value)
+		}
+
+		if strings.HasPrefix(key, "enable") {
+			toggleFlags = append(toggleFlags, fmt.Sprintf("%v=%v", key, value))
+		}
+
+		if strings.HasPrefix(key, "skip") {
+			skippingFlags = append(skippingFlags, fmt.Sprintf("%v=%v", key, value))
+		}
+	}
+
+	workflowInfo := fmt.Sprintf("Name: %v", color.HiCyanString(parsedFlow.Name)) + ", " + fmt.Sprintf("Total Steps: %v", color.HiCyanString("%v", totalSteps)) + ", " + fmt.Sprintf("Total Modules: %v", color.HiCyanString("%v", totalModules))
+	content = append(content, []string{
+		"Workflow Information", workflowInfo,
+	})
+	content = append(content, []string{
+		"Description", parsedFlow.Desc,
+	})
+
+	content = append(content, []string{
+		"Toggleable parameter", strings.Join(toggleFlags, ", "),
+	})
+
+	content = append(content, []string{
+		"Skippable parameter", strings.Join(skippingFlags, ", "),
+	})
+
+	if parsedFlow.Usage != "" {
+		content = append(content, []string{
+			"Examples Commands", strings.TrimSpace(parsedFlow.Usage),
+		})
+	}
+
+	table := tablewriter.NewWriter(os.Stdout)
+	table.SetRowLine(true)
+	table.SetBorders(tablewriter.Border{Left: true, Top: true, Right: true, Bottom: true})
+	table.SetColWidth(120)
+	table.SetAutoWrapText(false)
+	table.AppendBulk(content)
+	table.Render()
+
+	return nil
+}

core/backup.go

@@ -0,0 +1,68 @@
+package core
+
+import (
+	"os"
+	"path"
+	"strings"
+
+	"github.com/fatih/color"
+	"github.com/j3ssie/osmedeus/execution"
+	"github.com/j3ssie/osmedeus/libs"
+	"github.com/j3ssie/osmedeus/utils"
+)
+
+// in workflow file
+// Compress('{{Backup}}/{{Workspace}}.tar.gz', '{{Output}}')
+// Decompress('{{Output}}', '{{Backup}}/{{Workspace}}.tar.gz')
+
+func (r *Runner) BackupWorkspace() {
+	utils.InforF("Backing up the workspace: %v", r.Target["Workspace"])
+	outputDir := r.Target["Output"]
+	dest := path.Join(r.Opt.Env.BackupFolder, r.Target["Workspace"]) + ".tar.gz"
+	if utils.FileExists(dest) {
+		os.Remove(dest)
+	}
+
+	execution.Compress(dest, outputDir)
+	if utils.FileExists(dest) {
+		utils.GoodF("Backup workspace save at %s", color.HiMagentaString(dest))
+	}
+}
+
+func CompressWorkspace(target string, opt libs.Options) {
+	utils.InforF("Backing up the workspace: %v", color.HiCyanString(target))
+	outputDir := path.Join(opt.Env.WorkspacesFolder, target)
+	if utils.FolderLength(outputDir) == 0 {
+		utils.ErrorF("Workspace is empty: %s", outputDir)
+		return
+	}
+
+	dest := path.Join(opt.Env.BackupFolder, target) + ".tar.gz"
+	if utils.FileExists(dest) {
+		os.Remove(dest)
+	}
+
+	execution.Compress(dest, outputDir)
+	if utils.FileExists(dest) {
+		utils.InforF("The workspace has been backed up and saved in %s", color.HiMagentaString(dest))
+	}
+}
+
+func ExtractBackup(src string, opt libs.Options) {
+	if !utils.FileExists(src) {
+		utils.ErrorF("Backup file not found: %s", src)
+		return
+	}
+
+	target := strings.ReplaceAll(path.Base(src), ".tar.gz", "")
+	dest := path.Join(opt.Report.ExtractFolder, target)
+	if !strings.HasSuffix(dest, "/") {
+		dest += "/"
+	}
+
+	if utils.FolderExists(dest) {
+		utils.MakeDir(dest)
+	}
+	execution.Decompress(dest, src)
+	utils.GoodF("Extracting the %v to %s", color.HiCyanString(target), color.HiMagentaString(dest))
+}

core/banner.go

@@ -0,0 +1,63 @@
+package core
+
+import (
+	"github.com/fatih/color"
+	"github.com/j3ssie/osmedeus/libs"
+)
+
+// Banner print ascii banner
+func Banner() string {
+	version := color.HiWhiteString(libs.VERSION)
+	author := color.MagentaString(libs.AUTHOR)
+	//W := color.HiWhiteString(``)
+	b := color.GreenString(``)
+
+	b += color.GreenString(`
+            
+                                        .;1tfLCL1,
+                                       .,,..;i;f0G;
+                                             ,:,tCC.  ...
+                                             ;i:fCL,1LLtf1i;,
+                                           .,::tCL1LC1::;, .,,
+                                           ;1:tCL,tLt,1:
+                                          ,::tLf, 1Lf;::.
+                                        .ii:tLt.  .1Lf;i1.
+                                        ,:;tf1      1ft;::
+                                     .1;:tf1 `) + color.HiWhiteString(` ,i1t1, `) + color.GreenString(` ift;;1,
+                                    ,i:t;f.`) + color.HiWhiteString(` ,LLffLL: `) + color.GreenString(` tft;i:
+                                    .;:;fff `) + color.HiWhiteString(` .LCLLLf,`) + color.GreenString(` 1ffi:;.
+                                    :fi;Lff1.  `) + color.HiWhiteString(`,;;:`) + color.GreenString(`  ifffi;f;
+                                     .:::tCLLfi:,,:ifLfLt::;.
+                                      ,11:1CCCCCLLLLLLf1;1t:
+                                      .it;:;1fLLLLfft1;:;ti.
+                                         ,:;::;;;;;;;;;;,
+                                           .,::::::::,.
+	`)
+
+	//
+	//
+	//b += "\n\t" + color.GreenString(`                                      @@@@@@`)
+	//b += "\n\t" + color.GreenString(`                                    .@@'  '@@.`)
+	//b += "\n\t" + color.GreenString(`                                    :@      @:`)
+	//b += "\n\t" + color.GreenString(`                                    :@  %v:@`, W) + color.GreenString(`  @:`)
+	//b += "\n\t" + color.GreenString(`                                    :@  %v:@`, W) + color.GreenString(`  @:`)
+	//
+	//b += "\n\t" + color.GreenString(`                                    :@      @:`)
+	//b += "\n\t" + color.GreenString(`                                    '@@.  .@@'`)
+	//b += "\n\t" + color.GreenString(`                                      @@@@@@`)
+	//b += "\n\t" + color.GreenString(`                                        @@`)
+	//b += "\n\t" + color.HiCyanString(`                                     @  `) + color.GreenString(`@@`) + color.HiCyanString(`  @`)
+	//b += "\n\t" + color.HiWhiteString(`                                    +@@`) + color.GreenString(` @@ `) + color.HiWhiteString(` @@+`)
+	//b += "\n\t" + color.GreenString(`                                 @@:@#@,@@,@#@:@@`)
+	//b += "\n\t" + color.GreenString(`                                ;@+@@'#@@@@#'@@+@;`)
+	//b += "\n\t" + color.GreenString(`                                @+ #@@  @@  @@# +@`)
+	//b += "\n\t" + color.GreenString(`                               @@  @+'@@@@@@'+@  @@`)
+	//b += "\n\t" + color.GreenString(`                               @.  @   ;@@;   @  .@`)
+	//b += "\n\t" + color.BlueString(`                              #@  '@          @;  @#`)
+
+	b += "\n\n\t" + color.GreenString(`                  Osmedeus Next Generation %v`, version) + color.GreenString(` by %v`, author)
+	b += "\n\n" + color.HiCyanString(`	                    %s`, libs.DESC) + "\n"
+	b += "\n" + color.HiWhiteString(`                                            ¯\_(ツ)_/¯`) + "\n\n"
+	color.Unset()
+	return b
+}

core/config.go

@@ -0,0 +1,382 @@
+package core
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"runtime"
+	"strings"
+
+	"github.com/fatih/color"
+	"github.com/j3ssie/osmedeus/libs"
+	"github.com/j3ssie/osmedeus/provider"
+	"github.com/j3ssie/osmedeus/utils"
+	"github.com/spf13/cast"
+	"github.com/spf13/viper"
+)
+
+var v *viper.Viper
+
+// InitConfig Init the config
+func InitConfig(options *libs.Options) error {
+	// ~/.osmedeus
+	RootFolder := filepath.Dir(utils.NormalizePath(options.ConfigFile))
+	if !utils.FolderExists(RootFolder) {
+		if err := os.MkdirAll(RootFolder, 0750); err != nil {
+			return err
+		}
+	}
+
+	// Base folder
+	// ~/osmedeus-base
+	BaseFolder := utils.NormalizePath(options.Env.BaseFolder)
+	if !utils.FolderExists(BaseFolder) {
+		fmt.Printf("%v Base folder not found at: %v\n", color.RedString("[Panic]"), color.HiGreenString(BaseFolder))
+		fmt.Printf(color.HiYellowString("[!]")+" Consider running the installation script first: %v\n", color.HiGreenString("bash <(curl -fsSL %v)", libs.INSTALL))
+		fmt.Printf(color.HiYellowString("[!]")+" Or better visit the installation guide at: %v\n", color.HiMagentaString("https://docs.osmedeus.org/installation/"))
+		os.Exit(-1)
+	}
+
+	// load all the tokens
+	options.TokenConfigFile = path.Join(BaseFolder, "token/osm-var.yaml")
+	if !utils.FolderExists(path.Dir(options.TokenConfigFile)) {
+		utils.MakeDir(path.Dir(options.TokenConfigFile))
+	}
+
+	options.Env.WorkspacesFolder = utils.NormalizePath(options.Env.WorkspacesFolder)
+	if !utils.FolderExists(options.Env.WorkspacesFolder) {
+		utils.MakeDir(options.Env.WorkspacesFolder)
+	}
+
+	// init config
+	options.ConfigFile = utils.NormalizePath(options.ConfigFile)
+	v = viper.New()
+	v.SetConfigName("config")
+	v.SetConfigType("yaml")
+	v.AddConfigPath(options.ConfigFile)
+	v.AddConfigPath(path.Dir(options.ConfigFile))
+
+	err := v.ReadInConfig()
+	if err != nil {
+		secret := utils.GenHash(utils.RandomString(8) + utils.GetTS())[:32] // only 32 char
+		prefix := secret[len(secret)-20 : len(secret)-1]
+
+		// set some default config if config file doesn't exist
+		v.SetDefault("Server", map[string]string{
+			"bind":        "0.0.0.0:8000",
+			"cors":        "*",
+			"secret":      secret,
+			"prefix":      prefix,
+			"ui":          path.Join(RootFolder, "server/ui"),
+			"cert_file":   path.Join(RootFolder, "server/ssl/cert.pem"),
+			"key_file":    path.Join(RootFolder, "server/ssl/key.pem"),
+			"master_pass": "",
+		})
+
+		v.SetDefault("Tactic", map[string]any{
+			"default":    runtime.NumCPU() * 8, // 8,16,32,64
+			"aggressive": runtime.NumCPU() * 16,
+			"gently":     runtime.NumCPU() * 2,
+		})
+
+		v.SetDefault("Mics", map[string]string{
+			"docs": utils.GetOSEnv("OSM_DOCS", libs.DOCS),
+		})
+
+		// DB connection config
+		dbPath := utils.NormalizePath(path.Join(RootFolder, "sqlite.db"))
+		v.SetDefault("Database", map[string]string{
+			"db_host": utils.GetOSEnv("DB_HOST", "127.0.0.1"),
+			"db_port": utils.GetOSEnv("DB_PORT", "3306"),
+			"db_name": utils.GetOSEnv("DB_NAME", "osm-core"),
+			"db_user": utils.GetOSEnv("DB_USER", "root"),
+			"db_pass": utils.GetOSEnv("DB_PASS", ""),
+			// default will be file system
+			"db_path": utils.GetOSEnv("DB_PATH", dbPath),
+			// sqlite or mysql
+			"db_type": utils.GetOSEnv("DB_TYPE", "filesystem"),
+		})
+
+		// default user
+		password := utils.GenHash(utils.GetTS())[:15]
+		v.SetDefault("Client", map[string]string{
+			"username": "osmedeus",
+			"password": password,
+			"jwt":      "",
+			"dest":     "http://127.0.0.1:8000",
+		})
+
+		v.SetDefault("Environments", map[string]string{
+			// RootFolder --> ~/.osmedeus/
+			"storages":        path.Join(RootFolder, "storages"),
+			"backups":         path.Join(RootFolder, "backups"),
+			"provider_config": path.Join(RootFolder, "provider"),
+			"instances":       path.Join(RootFolder, "instances"),
+
+			// store all the result
+			"workspaces": options.Env.WorkspacesFolder,
+
+			// this update occasionally
+			// BaseFolder --> ~/osmedeus-base/
+			"workflows":    path.Join(BaseFolder, "workflow"),
+			"binaries":     path.Join(BaseFolder, "binaries"),
+			"data":         path.Join(BaseFolder, "data"),
+			"cloud_config": path.Join(BaseFolder, "cloud"),
+		})
+
+		if err := v.WriteConfigAs(options.ConfigFile); err != nil {
+			utils.ErrorF("Error writing config file: %s", err)
+		}
+
+		utils.InforF("Created a new configuration file at %s", color.HiCyanString(options.ConfigFile))
+	}
+
+	if isWritable, _ := utils.IsWritable(options.ConfigFile); isWritable {
+		utils.ErrorF("config file does not writable: %v", color.HiCyanString(options.ConfigFile))
+		utils.BlockF("fatal", "Make sure you are login as 'root user' if your installation done via root user")
+		os.Exit(-1)
+	}
+	SetupOSEnv(options)
+	return nil
+}
+
+func LoadConfig(options *libs.Options) *viper.Viper {
+	v = viper.New()
+	v.SetConfigName("config")
+	v.SetConfigType("yaml")
+	v.AddConfigPath(options.ConfigFile)
+	v.AddConfigPath(path.Dir(options.ConfigFile))
+
+	if err := v.ReadInConfig(); err != nil {
+		utils.ErrorF("Error reading config file, %s", err)
+	}
+	return v
+}
+
+func ParsingConfig(options *libs.Options) {
+	v = LoadConfig(options)
+	GetEnv(options)
+	GetServer(options)
+	GetClient(options)
+	SetupOpt(options)
+	// get the config for cloud provider
+	GetCloud(options)
+}
+
+// GetEnv get environment options
+func GetEnv(options *libs.Options) {
+	envs := v.GetStringMapString("Environments")
+
+	options.Env.BinariesFolder = utils.NormalizePath(envs["binaries"])
+	utils.MakeDir(options.Env.BinariesFolder)
+
+	if options.Env.WorkFlowsFolder != "" {
+		options.Env.WorkFlowsFolder = utils.NormalizePath(options.Env.WorkFlowsFolder)
+	}
+
+	options.Env.DataFolder = utils.NormalizePath(envs["data"])
+	utils.MakeDir(options.Env.DataFolder)
+	// ose folder
+	options.Env.OseFolder = path.Join(options.Env.BaseFolder, "ose")
+	utils.MakeDir(options.Env.DataFolder)
+	options.Env.BackupFolder = utils.NormalizePath(envs["backups"])
+	utils.MakeDir(options.Env.BackupFolder)
+	options.Env.UIFolder = path.Join(options.Env.BaseFolder, "ui")
+
+	// local data
+	options.Env.StoragesFolder = utils.NormalizePath(envs["storages"])
+	utils.MakeDir(options.Env.StoragesFolder)
+	options.Env.WorkspacesFolder = utils.NormalizePath(envs["workspaces"])
+	utils.MakeDir(options.Env.WorkspacesFolder)
+
+	customWorkflow := utils.GetOSEnv("CUSTOM_OSM_WORKFLOW", "CUSTOM_OSM_WORKFLOW")
+	if customWorkflow != "CUSTOM_OSM_WORKFLOW" && options.Env.WorkFlowsFolder == "" {
+		options.Env.WorkFlowsFolder = utils.NormalizePath(customWorkflow)
+	}
+
+	if options.Env.WorkFlowsFolder == "" {
+		options.Env.WorkFlowsFolder = utils.NormalizePath(envs["workflows"])
+	}
+
+	// @NOTE: well of course you can rebuild the core engine binary to bypass this check
+	// However, the premium package primarily focuses on exclusive workflow and specialized wordlists.
+	// see more about it here: https://docs.osmedeus.org/faq/#premium-package-related-questions
+	// and https://docs.osmedeus.org/premium/
+	if utils.FileExists(path.Join(options.Env.WorkFlowsFolder, "premium.md")) {
+		options.PremiumPackage = true
+	}
+
+	// cloud stuff
+
+	// ~/.osmedeus/providers/
+	options.Env.ProviderFolder = utils.NormalizePath(envs["provider_config"])
+	options.Env.InstancesFolder = utils.NormalizePath(envs["instances"])
+	if options.Env.InstancesFolder == "" {
+		options.Env.InstancesFolder = utils.NormalizePath(path.Join(options.Env.RootFolder, "instances"))
+	}
+	utils.MakeDir(options.Env.ProviderFolder)
+	utils.MakeDir(options.Env.InstancesFolder)
+
+	// ~/osmedeus-base/clouds/
+	options.Env.CloudConfigFolder = utils.NormalizePath(envs["cloud_config"])
+}
+
+// SetupOpt get storage repos
+func SetupOpt(options *libs.Options) {
+	// auto append PATH with Plugin folder
+	osPATH := utils.GetOSEnv("PATH", "PATH")
+	if !strings.Contains(osPATH, options.Env.BinariesFolder) {
+		utils.DebugF("Append $PATH with: %s", options.Env.BinariesFolder)
+		os.Setenv("PATH", fmt.Sprintf("%s:%s", osPATH, strings.TrimRight(options.Env.BinariesFolder, "/")))
+	}
+
+	tactics := v.GetStringMap("Tactic")
+	options.Tactics = strings.ToLower(options.Tactics)
+	options.ThreadsHold.Default = cast.ToInt(tactics["default"])
+	options.ThreadsHold.Aggressive = cast.ToInt(tactics["aggressive"])
+	options.ThreadsHold.Gently = cast.ToInt(tactics["gently"])
+
+	// try to autocorrect the tactic name
+	switch options.Tactics {
+	case "aggressive", "agg", "aggr", "aggrsive":
+		options.Tactics = "aggressive"
+	case "gently", "gen", "gent", "gentl":
+		options.Tactics = "gently"
+	}
+	defaultThreadsHold := cast.ToInt(tactics[options.Tactics])
+
+	if defaultThreadsHold == 0 {
+		utils.ErrorF("tactic %s not found, switching to the %v one", color.HiRedString(options.Tactics), color.HiYellowString("default"))
+		defaultThreadsHold = cast.ToInt(tactics["default"])
+		options.Tactics = "default"
+		// in case you're still using the old config
+		if defaultThreadsHold == 0 {
+			defaultThreadsHold = 4
+		}
+	}
+
+	// override if you put --threads-hold flag
+	if options.Threads == 0 {
+		options.Threads = defaultThreadsHold
+	}
+
+	/* some special conditions below */
+
+	// change {{.Storage}} from ~/.osmedeus/storages to ~/.osmedeus/destorages
+	if options.EnableDeStorage {
+		utils.DebugF("Dedicated Storage Enabled")
+		options.Env.StoragesFolder = options.Git.DeStorage
+		if !utils.FolderExists(options.Env.StoragesFolder) {
+			utils.MakeDir(options.Env.StoragesFolder)
+		}
+	}
+}
+
+func GetCloud(options *libs.Options) {
+	if !options.PremiumPackage {
+		return
+	}
+
+	// ~/osemedeus-base/cloud/provider.yaml
+	cloudConfigFile := path.Join(options.Env.CloudConfigFolder, "provider.yaml")
+
+	options.CloudConfigFile = cloudConfigFile
+	utils.DebugF("Parsing cloud config from: %s", color.HiCyanString(options.CloudConfigFile))
+	providerConfigs, err := provider.ParseProvider(options.CloudConfigFile)
+
+	if err != nil {
+		utils.InforF("💡 You can start the wizard with the command: %s", color.HiCyanString("%s provider wizard", libs.BINARY))
+	}
+
+	options.Cloud.BuildRepo = providerConfigs.Builder.BuildRepo
+	options.Cloud.SecretKey = utils.NormalizePath(providerConfigs.Builder.SecretKey)
+	options.Cloud.PublicKey = utils.NormalizePath(providerConfigs.Builder.PublicKey)
+	if options.Cloud.SecretKey == "" {
+		options.Cloud.SecretKey = path.Join(options.Env.CloudConfigFolder, "ssh/cloud")
+		options.Cloud.PublicKey = path.Join(options.Env.CloudConfigFolder, "ssh/cloud.pub")
+	}
+
+	// check SSH Keys
+	if !utils.FileExists(options.Cloud.SecretKey) {
+		keysDir := path.Dir(options.Cloud.SecretKey)
+		os.RemoveAll(keysDir)
+		utils.MakeDir(keysDir)
+
+		utils.InforF("Generate SSH Key at: %v", options.Cloud.SecretKey)
+		var err error
+		_, err = utils.RunCommandWithErr(fmt.Sprintf(`ssh-keygen -t ed25519 -f %s -q -N ''`, options.Cloud.SecretKey))
+		if err != nil {
+			color.Red("[-] error generated SSH Key for cloud config at: %v", options.Cloud.SecretKey)
+			return
+		}
+	}
+
+	if utils.FileExists(options.Cloud.SecretKey) {
+		utils.DebugF("Detected secret key: %v", options.Cloud.SecretKey)
+		utils.DebugF("Detected public key: %v", options.Cloud.PublicKey)
+		options.Cloud.SecretKeyContent = strings.TrimSpace(utils.GetFileContent(options.Cloud.SecretKey))
+		options.Cloud.PublicKeyContent = strings.TrimSpace(utils.GetFileContent(options.Cloud.PublicKey))
+	}
+}
+
+// GetServer get server options
+func GetServer(options *libs.Options) {
+	server := v.GetStringMapString("Server")
+
+	options.Server.Bind = server["bind"]
+	options.Server.Cors = server["cors"]
+	options.Server.JWTSecret = server["secret"]
+	options.Server.StaticPrefix = server["prefix"]
+	options.Server.UIPath = utils.NormalizePath(server["ui"])
+	utils.MakeDir(path.Dir(options.Server.UIPath))
+
+	options.Server.MasterPassword = server["master_pass"]
+	options.Server.CertFile = utils.NormalizePath(server["cert_file"])
+	options.Server.KeyFile = utils.NormalizePath(server["key_file"])
+	utils.MakeDir(path.Dir(options.Server.CertFile))
+
+	db := v.GetStringMapString("Database")
+
+	options.Server.DBPath = utils.NormalizePath(db["db_path"])
+	options.Server.DBType = db["db_type"]
+	// this should be remote one
+	if options.Server.DBType == "mysql" {
+		options.Server.DBUser = db["db_user"]
+		options.Server.DBPass = db["db_pass"]
+		options.Server.DBHost = db["db_host"]
+		options.Server.DBPort = db["db_port"]
+		options.Server.DBName = db["db_name"]
+
+		//  “user:password@/dbname?charset=utf8&parseTime=True&loc=Local”
+		cred := fmt.Sprintf("%v:%v", options.Server.DBUser, options.Server.DBPass)
+		dest := fmt.Sprintf("%v:%v", options.Server.DBHost, options.Server.DBPort)
+		dbURL := fmt.Sprintf("%v@tcp(%v)/%v?charset=utf8&parseTime=True&loc=Local", cred, dest, options.Server.DBName)
+		options.Server.DBConnection = dbURL
+	}
+}
+
+// GetClient get options for client
+func GetClient(options *libs.Options) map[string]string {
+	client := v.GetStringMapString("Client")
+	options.Client.Username = client["username"]
+	options.Client.Password = client["password"]
+	return client
+}
+
+func SetTactic(options *libs.Options) {
+	v = LoadConfig(options)
+	if err := v.ReadInConfig(); err != nil {
+		utils.ErrorF("Error reading config file, %s", err)
+	}
+
+	baseThreads := options.Threads
+	utils.InforF("Set base threads to %v", color.HiCyanString("%v", baseThreads))
+	v.Set("Tactic", map[string]any{
+		"default":    baseThreads, // 2,4,8,16
+		"aggressive": baseThreads * 4,
+		"gently":     int(baseThreads / 2),
+	})
+
+	v.WriteConfig()
+}

core/cron.go

@@ -0,0 +1,30 @@
+package core
+
+import (
+	"github.com/fatih/color"
+	"github.com/j3ssie/osmedeus/utils"
+	"github.com/jasonlvhit/gocron"
+	"github.com/spf13/cast"
+)
+
+func taskWithParams(cmd string) {
+	utils.InforF("Exec: %v", color.HiMagentaString(cmd))
+	_, err := utils.RunCommandSteamOutput(cmd)
+	if err != nil {
+		utils.ErrorF("Error running command: %v", err)
+	}
+}
+
+func RunCron(cmd string, schedule int) {
+
+	if schedule == -1 {
+		utils.InforF("Run command forever: %v", cmd)
+		for {
+			taskWithParams(cmd)
+		}
+	}
+
+	utils.InforF("Start cron job with %v seconds: %v", schedule, color.HiCyanString(cmd))
+	gocron.Every(cast.ToUint64(schedule)).Minutes().Do(taskWithParams, cmd)
+	<-gocron.Start()
+}

core/db.go

@@ -0,0 +1,229 @@
+package core
+
+import (
+	"os"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/fatih/color"
+	"github.com/j3ssie/osmedeus/database"
+	"github.com/j3ssie/osmedeus/libs"
+	"github.com/j3ssie/osmedeus/utils"
+	jsoniter "github.com/json-iterator/go"
+	"github.com/robertkrimen/otto"
+)
+
+func (r *Runner) LoadDBScripts() string {
+	var output string
+
+	r.VM.Set(TotalSubdomain, func(call otto.FunctionCall) otto.Value {
+		length := utils.FileLength(call.Argument(0).String())
+		r.TargetObj.TotalAssets = length
+		utils.InforF("Total subdomain found: %v", color.HiMagentaString("%v", length))
+		return otto.Value{}
+	})
+
+	r.VM.Set(TotalDns, func(call otto.FunctionCall) otto.Value {
+		length := utils.FileLength(call.Argument(0).String())
+		r.TargetObj.TotalDns = length
+		utils.InforF("Total Dns: %v", color.HiMagentaString("%v", length))
+		return otto.Value{}
+	})
+
+	r.VM.Set(TotalScreenShot, func(call otto.FunctionCall) otto.Value {
+		length := utils.FileLength(call.Argument(0).String())
+		r.TargetObj.TotalScreenShot = length
+		utils.InforF("Total ScreenShot: %v", color.HiMagentaString("%v", length))
+		return otto.Value{}
+	})
+
+	r.VM.Set(TotalTech, func(call otto.FunctionCall) otto.Value {
+		length := utils.FileLength(call.Argument(0).String())
+		r.TargetObj.TotalTech = length
+		utils.InforF("Total Tech: %v", color.HiMagentaString("%v", length))
+		return otto.Value{}
+	})
+
+	r.VM.Set(TotalVulnerability, func(call otto.FunctionCall) otto.Value {
+		data := utils.ReadingFileUnique(call.Argument(0).String())
+		var length int
+		for _, line := range data {
+			if !strings.Contains(line, "-info") {
+				length += 1
+			}
+		}
+		r.TargetObj.TotalVulnerability = length
+		utils.InforF("Total Vulnerability: %v", color.HiMagentaString("%v", length))
+		return otto.Value{}
+	})
+
+	r.VM.Set(TotalArchive, func(call otto.FunctionCall) otto.Value {
+		length := utils.FileLength(call.Argument(0).String())
+		r.TargetObj.TotalArchive = length
+		utils.InforF("Total Archive: %v", color.HiMagentaString("%v", length))
+		return otto.Value{}
+	})
+
+	r.VM.Set(TotalLink, func(call otto.FunctionCall) otto.Value {
+		length := utils.FileLength(call.Argument(0).String())
+		r.TargetObj.TotalLink = length
+		utils.InforF("Total Link: %v", color.HiMagentaString("%v", length))
+		return otto.Value{}
+	})
+
+	r.VM.Set(TotalDirb, func(call otto.FunctionCall) otto.Value {
+		length := utils.FileLength(call.Argument(0).String())
+		r.TargetObj.TotalDirb = length
+		utils.InforF("Total Dirb: %v", color.HiMagentaString("%v", length))
+		return otto.Value{}
+	})
+
+	r.VM.Set(CreateReport, func(call otto.FunctionCall) otto.Value {
+		args := call.ArgumentList
+		reportPath := args[0].String()
+		if utils.FileExists(reportPath) {
+			return otto.Value{}
+		}
+		moduleName := "inline"
+		if len(args) > 1 {
+			moduleName = args[1].String()
+		}
+		reportItem := database.Report{
+			ReportPath: reportPath,
+			Module:     moduleName,
+		}
+		r.TargetObj.Reports = append(r.TargetObj.Reports, reportItem)
+		return otto.Value{}
+	})
+
+	return output
+}
+
+func (r *Runner) DBNewTarget() {
+	r.TargetObj = database.Target{
+		InputName: r.Input,
+		Workspace: r.Workspace,
+		InputType: r.InputType,
+	}
+
+	r.DBRuntimeUpdate()
+}
+
+func (r *Runner) DBNewScan() {
+	r.ScanObj = database.Scan{
+		TaskType: r.RoutineType,
+		TaskName: path.Base(r.RoutineName),
+
+		TotalSteps: r.TotalSteps,
+		InputName:  r.Input,
+		InputType:  r.InputType,
+
+		MarkDownSunmmary: path.Join(r.WorkspaceFolder, "summary.md"),
+		MarkDownReport:   path.Join(r.WorkspaceFolder, "summary.html"),
+
+		LogFile:    r.Opt.LogFile,
+		Target:     r.TargetObj,
+		ProcessID:  os.Getpid(),
+		IsRunning:  true,
+		IsDone:     false,
+		IsPrepared: true,
+		IsStarted:  true,
+	}
+
+	if r.RunnerType == "cloud" {
+		r.ScanObj.IsCloud = true
+	}
+
+	if r.Opt.Scan.RemoteCall {
+		r.ScanObj.IsCloud = true
+	}
+
+	r.ScanObj.CreatedAt = time.Now()
+
+	if runtimeData, err := jsoniter.MarshalToString(r.ScanObj); err == nil {
+		utils.WriteToFile(r.RuntimeFile, runtimeData)
+	}
+
+}
+
+func (r *Runner) DBUpdateScan() {
+	r.ScanObj.DoneStep = r.DoneStep
+	r.ScanObj.CurrentModule = r.CurrentModule
+	r.ScanObj.RunningTime = r.RunningTime
+	r.ScanObj.ProcessID = os.Getpid()
+
+	if r.ScanObj.DoneStep == r.ScanObj.TotalSteps {
+		r.ScanObj.IsDone = true
+		r.ScanObj.IsRunning = false
+	} else {
+		r.ScanObj.IsRunning = true
+		r.ScanObj.IsDone = false
+
+	}
+
+	utils.DebugF("[DB] Finished %v steps in the %v module", color.HiCyanString("%v/%v", r.DoneStep, r.TotalSteps), r.CurrentModule)
+	r.DBRuntimeUpdate()
+}
+
+func (r *Runner) DBDoneScan() {
+	r.ScanObj.CurrentModule = "done"
+	r.ScanObj.RunningTime = r.RunningTime
+
+	r.ScanObj.DoneStep = r.TotalSteps
+	r.ScanObj.IsDone = true
+	r.ScanObj.IsRunning = false
+	r.ScanObj.IsStarted = false
+	r.ScanObj.UpdatedAt = time.Now()
+
+	utils.DebugF("[DB] The scan has been completed: %v -- %v", color.HiCyanString(r.ScanObj.InputName), color.HiCyanString(r.ScanObj.TaskName))
+	if runtimeData, err := jsoniter.MarshalToString(r.ScanObj); err == nil {
+		utils.WriteToFile(r.DoneFile, runtimeData)
+		utils.WriteToFile(r.RuntimeFile, runtimeData)
+	}
+
+	if utils.FileExists(r.ScanObj.MarkDownReport) {
+		utils.InforF("Markdown summary has been generated at: %v", color.GreenString(r.ScanObj.MarkDownReport))
+	}
+	if utils.FileExists(r.ScanObj.MarkDownSunmmary) {
+		utils.InforF("HTML summary has been generated at: %v", color.GreenString(r.ScanObj.MarkDownSunmmary))
+	}
+}
+
+func (r *Runner) DBRuntimeUpdate() {
+	r.ScanObj.UpdatedAt = time.Now()
+	r.ScanObj.Target = r.TargetObj
+	if runtimeData, err := jsoniter.MarshalToString(r.ScanObj); err == nil {
+		utils.WriteToFile(r.RuntimeFile, runtimeData)
+	}
+}
+
+func (r *Runner) DBNewReports(module libs.Module) {
+	r.ScanObj.CurrentModule = r.CurrentModule
+	r.ScanObj.RunningTime = r.RunningTime
+
+	var reports []string
+	reports = append(reports, module.Report.Final...)
+	reports = append(reports, module.Report.Noti...)
+	reports = append(reports, module.Report.Diff...)
+
+	utils.DebugF("Updating %v report records", len(reports))
+	for _, report := range reports {
+		reportType := "text"
+		if strings.HasSuffix(report, ".html") {
+			reportType = "html"
+		}
+
+		reportObj := database.Report{
+			ReportName: path.Base(report),
+			ModulePath: module.ModulePath,
+			Module:     module.Name,
+			ReportPath: report,
+			ReportType: reportType,
+		}
+
+		r.TargetObj.Reports = append(r.TargetObj.Reports, reportObj)
+
+	}
+	r.ScanObj.Target = r.TargetObj
+}

core/external.go

@@ -0,0 +1,298 @@
+package core
+
+import (
+	"fmt"
+	"path"
+	"time"
+
+	"github.com/j3ssie/osmedeus/execution"
+	"github.com/j3ssie/osmedeus/utils"
+	"github.com/robertkrimen/otto"
+)
+
+func (r *Runner) LoadExternalScripts() string {
+	var output string
+	vm := r.VM
+
+	// special scripts
+	vm.Set(Cleaning, func(call otto.FunctionCall) otto.Value {
+		if r.Opt.NoClean {
+			utils.InforF("Disabled Cleaning")
+			return otto.Value{}
+		}
+		execution.Cleaning(call.Argument(0).String(), r.Reports)
+		return otto.Value{}
+	})
+
+	// scripts for cleaning modules
+	vm.Set(CleanAmass, func(call otto.FunctionCall) otto.Value {
+		src := call.Argument(0).String()
+		dest := call.Argument(1).String()
+		execution.CleanAmass(src, dest)
+		return otto.Value{}
+	})
+
+	vm.Set(CleanRustScan, func(call otto.FunctionCall) otto.Value {
+		src := call.Argument(0).String()
+		dest := call.Argument(1).String()
+		execution.CleanRustScan(src, dest)
+		return otto.Value{}
+	})
+
+	vm.Set(CleanGoBuster, func(call otto.FunctionCall) otto.Value {
+		src := call.Argument(0).String()
+		dest := call.Argument(1).String()
+		execution.CleanGoBuster(src, dest)
+		return otto.Value{}
+	})
+	vm.Set(CleanMassdns, func(call otto.FunctionCall) otto.Value {
+		src := call.Argument(0).String()
+		dest := call.Argument(1).String()
+		execution.CleanMassdns(src, dest)
+		return otto.Value{}
+	})
+
+	vm.Set(CleanSWebanalyze, func(call otto.FunctionCall) otto.Value {
+		src := call.Argument(0).String()
+		dest := call.Argument(1).String()
+		execution.CleanSWebanalyze(src, dest)
+		return otto.Value{}
+	})
+	vm.Set(CleanJSONDnsx, func(call otto.FunctionCall) otto.Value {
+		src := call.Argument(0).String()
+		dest := call.Argument(1).String()
+		execution.CleanJSONDnsx(src, dest)
+		return otto.Value{}
+	})
+
+	vm.Set(CleanJSONHttpx, func(call otto.FunctionCall) otto.Value {
+		src := call.Argument(0).String()
+		dest := call.Argument(1).String()
+		execution.CleanJSONHttpx(src, dest)
+		return otto.Value{}
+	})
+
+	// Deprecated
+	vm.Set(CleanWebanalyze, func(call otto.FunctionCall) otto.Value {
+		src := call.Argument(0).String()
+		dest := call.Argument(1).String()
+		args := call.ArgumentList
+
+		techSum := path.Join(path.Dir(dest), fmt.Sprintf("tech-overview-%v.txt", r.Target["Workspace"]))
+		if len(args) > 3 {
+			techSum = args[2].String()
+		}
+		execution.CleanWebanalyze(src, dest, techSum)
+		return otto.Value{}
+	})
+
+	vm.Set(CleanArjun, func(call otto.FunctionCall) otto.Value {
+		// src mean folder contain arjun output
+		src := call.Argument(0).String()
+		dest := call.Argument(1).String()
+		execution.CleanArjun(src, dest)
+		return otto.Value{}
+	})
+
+	vm.Set(CleanFFUFJson, func(call otto.FunctionCall) otto.Value {
+		src := call.Argument(0).String()
+		dest := call.Argument(1).String()
+		execution.CleanFFUFJson(src, dest)
+		return otto.Value{}
+	})
+
+	vm.Set(GenNucleiReport, func(call otto.FunctionCall) otto.Value {
+		src := call.Argument(0).String()
+		dest := call.Argument(1).String()
+		args := call.ArgumentList
+
+		templateFile := ""
+		if len(args) >= 3 {
+			templateFile = args[2].String()
+		}
+		execution.GenNucleiReport(r.Opt, src, dest, templateFile)
+		return otto.Value{}
+	})
+
+	return output
+}
+
+func (r *Runner) LoadGitScripts() string {
+	var output string
+	vm := r.VM
+	options := r.Opt
+
+	// Clone("git@xxx.git", "/tmp/dest")
+	vm.Set(Clone, func(call otto.FunctionCall) otto.Value {
+		execution.GitClone(call.Argument(0).String(), call.Argument(1).String(), false, options)
+		return otto.Value{}
+	})
+	// like clone but delete the destination folder first
+	vm.Set(FClone, func(call otto.FunctionCall) otto.Value {
+		execution.GitClone(call.Argument(0).String(), call.Argument(1).String(), true, options)
+		return otto.Value{}
+	})
+
+	vm.Set(PushResult, func(call otto.FunctionCall) otto.Value {
+		for folder := range options.Storages {
+			execution.PullResult(folder, options)
+			time.Sleep(3 * time.Second)
+			execution.PullResult(folder, options)
+			commitMess := fmt.Sprintf("%v|%v|%v", options.Module.Name, options.Scan.ROptions["Workspace"], utils.GetCurrentDay())
+			execution.PushResult(folder, commitMess, options)
+		}
+		return otto.Value{}
+	})
+	// push result but specific folder
+	vm.Set(PushFolder, func(call otto.FunctionCall) otto.Value {
+		folder := call.Argument(0).String()
+		execution.PullResult(folder, options)
+		time.Sleep(3 * time.Second)
+		execution.PullResult(folder, options)
+		commitMess := fmt.Sprintf("%v|%v|%v", options.Module.Name, options.Scan.ROptions["Workspace"], utils.GetCurrentDay())
+		execution.PushResult(folder, commitMess, options)
+		return otto.Value{}
+	})
+
+	// push result but specific folder
+	vm.Set(PullFolder, func(call otto.FunctionCall) otto.Value {
+		folder := call.Argument(0).String()
+		execution.PullResult(folder, options)
+		time.Sleep(3 * time.Second)
+		execution.PullResult(folder, options)
+		return otto.Value{}
+	})
+
+	vm.Set(DiffCompare, func(call otto.FunctionCall) otto.Value {
+		src := call.Argument(0).String()
+		dest := call.Argument(1).String()
+		output := call.Argument(2).String()
+		execution.DiffCompare(src, dest, output, options)
+		return otto.Value{}
+	})
+
+	vm.Set(GitDiff, func(call otto.FunctionCall) otto.Value {
+		args := call.ArgumentList
+		src := args[0].String()
+		output := call.Argument(1).String()
+		history := "1"
+		if len(args) < 2 {
+			history = call.Argument(2).String()
+		}
+		execution.GitDiff(src, output, history, options)
+		return otto.Value{}
+	})
+	vm.Set(LoopGitDiff, func(call otto.FunctionCall) otto.Value {
+		args := call.ArgumentList
+		src := args[0].String()
+		output := call.Argument(1).String()
+		execution.LoopGitDiff(src, output, options)
+		return otto.Value{}
+	})
+
+	/* --- CDN S3 Bucket --- */
+
+	// UploadToS3("/tmp/src", "xxx/xxx") or UploadToS3("/tmp/src", "your-cdn.s3.ap-southeast-1.amazonaws.com")
+	vm.Set(UploadToS3, func(call otto.FunctionCall) otto.Value {
+		args := call.ArgumentList
+		bucket := ""
+		src := args[0].String()
+		if len(args) > 1 {
+			bucket = args[1].String()
+		}
+
+		// if bucket is empty, use the default one
+		if bucket == "" {
+			bucket = options.Cdn.Bucket
+		}
+
+		execution.UploadToS3(options, src, bucket)
+		return otto.Value{}
+	})
+
+	// DownloadFromS3("xxx", "/tmp/dest")
+	vm.Set(DownloadFromS3, func(call otto.FunctionCall) otto.Value {
+		args := call.ArgumentList
+		bucket := ""
+		src := args[0].String()
+		dest := args[1].String()
+		if len(args) > 2 {
+			bucket = args[2].String()
+		}
+
+		// if bucket is empty, use the default one
+		if bucket == "" {
+			bucket = options.Cdn.Bucket
+		}
+
+		execution.DownloadFromS3(options, src, dest, bucket)
+		return otto.Value{}
+	})
+
+	// DownloadFile("https://xxx.com", "/tmp/dest")
+	vm.Set(DownloadFile, func(call otto.FunctionCall) otto.Value {
+		args := call.ArgumentList
+		src := args[0].String()
+		output := args[1].String()
+		execution.DownloadFile(options, src, output)
+		return otto.Value{}
+	})
+
+	/* --- end CDN S3 Bucket --- */
+
+	/* --- Gitlab API --- */
+
+	// CreateRepo("repo-name")
+	// CreateRepo("repo-name", "tags")
+	vm.Set(CreateRepo, func(call otto.FunctionCall) otto.Value {
+		args := call.ArgumentList
+		repoName := args[0].String()
+		tags := ""
+		if len(args) > 1 {
+			tags = args[1].String()
+		}
+		execution.CreateGitlabRepo(repoName, tags, options)
+		return otto.Value{}
+	})
+
+	vm.Set(DeleteRepo, func(call otto.FunctionCall) otto.Value {
+		args := call.ArgumentList
+		repoName := args[0].String()
+		execution.DeleteRepo(repoName, 0, options)
+		return otto.Value{}
+	})
+	vm.Set(DeleteRepoByPid, func(call otto.FunctionCall) otto.Value {
+		args := call.ArgumentList
+		pid, err := args[0].ToInteger()
+		if err != nil {
+			return otto.Value{}
+		}
+		execution.DeleteRepo("", int(pid), options)
+		return otto.Value{}
+	})
+	vm.Set(ListProjects, func(call otto.FunctionCall) otto.Value {
+		args := call.ArgumentList
+		if len(args) > 0 {
+			uid, err := args[0].ToInteger()
+			if err == nil {
+				execution.ListProjects(int(uid), options)
+			}
+			return otto.Value{}
+		}
+		execution.ListProjects(0, options)
+		return otto.Value{}
+	})
+
+	/* --- end Gitlab API --- */
+
+	// GenMarkdownReport("markdown.md", "output.html")
+	vm.Set(GenMarkdownReport, func(call otto.FunctionCall) otto.Value {
+		args := call.ArgumentList
+		markdownFile := args[0].String()
+		outputHTML := args[1].String()
+		r.GenMarkdownReport(markdownFile, outputHTML)
+		return otto.Value{}
+	})
+
+	return output
+}

core/flow.go

@@ -0,0 +1,222 @@
+package core
+
+import (
+	"fmt"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"github.com/fatih/color"
+	"github.com/j3ssie/osmedeus/utils"
+	"github.com/thoas/go-funk"
+
+	"github.com/j3ssie/osmedeus/libs"
+)
+
+func ListAllFlowName(options libs.Options) (result []string) {
+	rawWorkflows := ListModules(options)
+	for _, item := range rawWorkflows {
+		result = append(result, strings.ReplaceAll(filepath.Base(item), ".yaml", ""))
+	}
+	return result
+}
+
+func ListModuleName(options libs.Options) (result []string) {
+	rawResult := DefaultWorkflows(options)
+	for _, item := range rawResult {
+		result = append(result, strings.ReplaceAll(filepath.Base(item), ".yaml", ""))
+	}
+	return result
+}
+
+// ListFlow list all available mode
+func ListFlow(options libs.Options) (result []string) {
+	modePath := path.Join(options.Env.WorkFlowsFolder, "/*.yaml")
+	result, err := filepath.Glob(modePath)
+	if err != nil {
+		return result
+	}
+	return result
+}
+
+// SelectFlow select flow to run
+func SelectFlow(flowName string, options libs.Options) []string {
+	flows := ListFlow(options)
+	var selectedFlow []string
+
+	// absolute path like -f customflows/general.yaml
+	if strings.HasSuffix(flowName, ".yaml") {
+		if utils.FileExists(flowName) {
+			selectedFlow = append(selectedFlow, flowName)
+			return selectedFlow
+		}
+	}
+
+	// -f test
+	if !strings.Contains(flowName, ",") {
+		selectedFlow = append(selectedFlow, singleMode(flowName, flows)...)
+	}
+
+	// -f test1,test2
+	flowNames := strings.Split(flowName, ",")
+	for _, item := range flowNames {
+		selectedFlow = append(selectedFlow, singleMode(item, flows)...)
+	}
+
+	// default custom flow folder
+	if !utils.FileExists(flowName) {
+		flowName = path.Join(options.Env.WorkFlowsFolder, "default-flows", flowName)
+		if utils.FileExists(flowName) {
+			selectedFlow = append(selectedFlow, flowName)
+		} else if utils.FileExists(flowName + ".yaml") {
+			flowName = flowName + ".yaml"
+			selectedFlow = append(selectedFlow, flowName)
+		}
+	}
+
+	selectedFlow = funk.UniqString(selectedFlow)
+	return selectedFlow
+}
+
+func singleMode(modeName string, modes []string) (selectedMode []string) {
+	for _, mode := range modes {
+		basemodeName := strings.TrimRight(strings.TrimRight(filepath.Base(mode), "yaml"), ".")
+		// select workflow file in workflow directory
+		if strings.ToLower(basemodeName) == strings.ToLower(modeName) {
+			selectedMode = append(selectedMode, mode)
+		}
+	}
+	return selectedMode
+}
+
+// ListModules list all available module
+func ListModules(options libs.Options) (modules []string) {
+	modePath := path.Join(options.Env.WorkFlowsFolder, "general/*.yaml")
+	if options.Flow.Type != "" {
+		modePath = path.Join(options.Env.WorkFlowsFolder, fmt.Sprintf("%v/*.yaml", options.Flow.Type))
+	}
+	if strings.HasSuffix(options.Scan.Flow, ".yaml") {
+		if options.Flow.Type == "" {
+			options.Flow.Type = "general"
+		}
+		modePath = path.Join(path.Dir(options.Scan.Flow), options.Flow.Type) + "/*.yaml"
+	}
+	modules, err := filepath.Glob(modePath)
+	if err != nil {
+		return modules
+	}
+	return modules
+}
+
+// SelectModules return list of modules name
+func SelectModules(moduleNames []string, options libs.Options) []string {
+	if strings.Contains(options.Flow.Type, "{{.") {
+		options.Flow.Type = ResolveData(options.Flow.Type, options.Scan.ROptions)
+	}
+	modules := ListModules(options)
+	var selectedModules []string
+	for _, item := range moduleNames {
+		selectedModules = append(selectedModules, singleSelectModule(item, modules)...)
+	}
+	selectedModules = funk.UniqString(selectedModules)
+
+	utils.DebugF("Select module name %v: %v", color.HiCyanString("%v", moduleNames), selectedModules)
+	return selectedModules
+}
+
+func singleSelectModule(moduleName string, modules []string) (selectedModules []string) {
+	for _, module := range modules {
+		baseModuleName := strings.Trim(strings.TrimRight(filepath.Base(module), "yaml"), ".")
+		if strings.ToLower(baseModuleName) == strings.ToLower(moduleName) {
+			selectedModules = append(selectedModules, module)
+		}
+	}
+	return selectedModules
+}
+
+// DefaultWorkflows select module from ~/.osmedeus/core/workflow/plugins/
+func DefaultWorkflows(options libs.Options) []string {
+	defaultModule := path.Join(options.Env.WorkFlowsFolder, "default-modules")
+	modePath := path.Join(defaultModule, "/*.yaml")
+	results, err := filepath.Glob(modePath)
+	if err != nil {
+		utils.ErrorF("No default module found in %v", defaultModule)
+		return []string{}
+	}
+	return results
+}
+
+// DirectSelectModule select module from ~/osmedeus-base/workflow/default-modules
+func DirectSelectModule(options libs.Options, moduleName string) string {
+	// got absolutely path
+	if utils.FileExists(moduleName) {
+		return moduleName
+	}
+
+	// select in cloud folder first if we're running the cloud scan
+	// ~/.osmedeus/core/workflow/cloud-modules/
+	basePlugin := path.Join(options.Env.WorkFlowsFolder, "cloud-modules")
+	modulePath := path.Join(basePlugin, moduleName)
+	if utils.FileExists(modulePath) {
+		utils.DebugF("Load module path: %v", modulePath)
+		return modulePath
+	}
+
+	modulePath = path.Join(basePlugin, moduleName+".yaml")
+	if utils.FileExists(modulePath) {
+		utils.DebugF("Load module path: %v", modulePath)
+		return modulePath
+	}
+
+	// ~/.osmedeus/core/workflow/default-modules/
+	basePlugin = path.Join(options.Env.WorkFlowsFolder, "default-modules")
+	modulePath = path.Join(basePlugin, moduleName)
+	utils.DebugF("Load module path: %v", modulePath)
+	if utils.FileExists(modulePath) {
+		return modulePath
+	}
+
+	modulePath = path.Join(basePlugin, moduleName+".yaml")
+	utils.DebugF("Load module path: %v", modulePath)
+	if utils.FileExists(modulePath) {
+		return modulePath
+	}
+	utils.DebugF("No plugin found with: %v", moduleName)
+	return ""
+}
+
+// ListScripts list all available mode
+func ListScripts(options libs.Options) (result []string) {
+	modePath := path.Join(options.Env.OseFolder, "/*.js")
+	result, err := filepath.Glob(modePath)
+	if err != nil {
+		return result
+	}
+
+	modePath = path.Join(options.Env.OseFolder, "/*/*.js")
+	DepthResult, err := filepath.Glob(modePath)
+	if err == nil {
+		result = append(result, DepthResult...)
+	}
+	return result
+}
+
+func SelectScript(scriptName string, options libs.Options) string {
+	scripts := ListScripts(options)
+	for _, script := range scripts {
+		if strings.Contains(scriptName, "/") {
+			if strings.HasSuffix(script, scriptName) || strings.HasSuffix(script, scriptName+".js") {
+				return script
+			}
+		}
+
+		compareName := path.Base(script)
+		if compareName == scriptName {
+			return script
+		}
+		if compareName == fmt.Sprintf("%s.js", scriptName) {
+			return script
+		}
+	}
+	return ""
+}

core/import.go

@@ -0,0 +1,574 @@
+package core
+
+// func (r *Runner) LoadImportScripts() string {
+// 	var output string
+
+// 	// DB scripts
+
+// 	r.VM.Set("ImportSubdomain", func(call otto.FunctionCall) otto.Value {
+// 		src := call.Argument(0).String()
+// 		r.ImportSubdomain(src)
+// 		return otto.Value{}
+// 	})
+
+// 	r.VM.Set("ImportDns", func(call otto.FunctionCall) otto.Value {
+// 		src := call.Argument(0).String()
+// 		r.ImportDns(src)
+// 		return otto.Value{}
+// 	})
+
+// 	r.VM.Set("ImportTech", func(call otto.FunctionCall) otto.Value {
+// 		src := call.Argument(0).String()
+// 		r.ImportTech(src)
+// 		return otto.Value{}
+// 	})
+
+// 	r.VM.Set("ImportHTTPJson", func(call otto.FunctionCall) otto.Value {
+// 		src := call.Argument(0).String()
+// 		r.ImportHTTPJson(src)
+// 		return otto.Value{}
+// 	})
+
+// 	r.VM.Set("ImportScreenShotJson", func(call otto.FunctionCall) otto.Value {
+// 		src := call.Argument(0).String()
+// 		r.ImportScreenShotJson(src)
+// 		return otto.Value{}
+// 	})
+
+// 	r.VM.Set("ImportPortJson", func(call otto.FunctionCall) otto.Value {
+// 		src := call.Argument(0).String()
+// 		r.ImportPortJson(src)
+// 		return otto.Value{}
+// 	})
+
+// 	r.VM.Set("ImportJaelesVulnJson", func(call otto.FunctionCall) otto.Value {
+// 		src := call.Argument(0).String()
+// 		r.ImportJaelesVulnJson(src)
+// 		return otto.Value{}
+// 	})
+
+// 	r.VM.Set("ImportNucleiVulnJson", func(call otto.FunctionCall) otto.Value {
+// 		src := call.Argument(0).String()
+// 		r.ImportNucleiVulnJson(src)
+// 		return otto.Value{}
+// 	})
+
+// 	r.VM.Set("ImportDirectoryJson", func(call otto.FunctionCall) otto.Value {
+// 		src := call.Argument(0).String()
+// 		r.ImportDirectoryJson(src)
+// 		return otto.Value{}
+// 	})
+
+// 	r.VM.Set("ImportLinks", func(call otto.FunctionCall) otto.Value {
+// 		src := call.Argument(0).String()
+// 		r.ImportLinks(src)
+// 		return otto.Value{}
+// 	})
+
+// 	r.VM.Set("ImportArchive", func(call otto.FunctionCall) otto.Value {
+// 		src := call.Argument(0).String()
+// 		r.ImportArchive(src)
+// 		return otto.Value{}
+// 	})
+
+// 	r.VM.Set("ImportIPRange", func(call otto.FunctionCall) otto.Value {
+// 		src := call.Argument(0).String()
+// 		r.ImportIPRange(src)
+// 		return otto.Value{}
+// 	})
+
+// 	r.VM.Set("ImportCred", func(call otto.FunctionCall) otto.Value {
+// 		src := call.Argument(0).String()
+// 		r.ImportCred(src)
+// 		return otto.Value{}
+// 	})
+
+// 	r.VM.Set("ImportCert", func(call otto.FunctionCall) otto.Value {
+// 		src := call.Argument(0).String()
+// 		r.ImportCert(src)
+// 		return otto.Value{}
+// 	})
+
+// 	r.VM.Set("ImportCloudBrute", func(call otto.FunctionCall) otto.Value {
+// 		src := call.Argument(0).String()
+// 		r.ImportCloudBrute(src)
+// 		return otto.Value{}
+// 	})
+
+// 	r.VM.Set("SummaryTarget", func(call otto.FunctionCall) otto.Value {
+// 		r.ScanObj.SummaryTarget()
+// 		return otto.Value{}
+// 	})
+
+// 	return output
+// }
+
+// func (r *Runner) ImportSubdomain(src string) {
+// 	if !utils.FileExists(src) {
+// 		utils.ErrorF("file not found: %v", src)
+// 		return
+// 	}
+
+// 	domains := utils.ReadingLines(src)
+// 	var objs []database.Asset
+
+// 	for _, domain := range domains {
+// 		domain = strings.TrimSpace(domain)
+// 		if domain == "" {
+// 			continue
+// 		}
+
+// 		obj := database.Asset{
+// 			AssetValue: domain,
+// 			//Dns:           nil,
+// 			HTTP:          nil,
+// 			Directory:     nil,
+// 			Vulnerability: nil,
+// 			ScanRefer:     r.ScanObj.ID,
+// 			TargetRefer:   r.ScanObj.TargetRefer,
+// 		}
+// 		objs = append(objs, obj)
+// 	}
+// 	database.ImportAssets(objs)
+// }
+
+// func (r *Runner) ImportDns(src string) {
+// 	if !utils.FileExists(src) {
+// 		utils.ErrorF("file not found: %v", src)
+// 		return
+// 	}
+
+// 	content := utils.ReadingLines(src)
+// 	for _, line := range content {
+// 		line = strings.TrimSpace(line)
+// 		if line == "" || !strings.Contains(line, " ") {
+// 			continue
+// 		}
+
+// 		raw := strings.Split(line, " ")
+// 		domain := strings.Trim(raw[0], ".")
+// 		dnsType := strings.TrimSpace(raw[1])
+// 		dnsValue := strings.Trim(raw[2], ".")
+
+// 		if strings.TrimSpace(domain) == "" || strings.TrimSpace(dnsValue) == "" {
+// 			continue
+// 		}
+
+// 		dnsChecksum := utils.GenHash(fmt.Sprintf("%s-%s-%s", domain, dnsType, dnsValue))
+
+// 		obj := database.Dns{
+// 			Domain:      domain,
+// 			DnsType:     dnsType,
+// 			DnsValue:    dnsValue,
+// 			DnsChecksum: dnsChecksum,
+// 			ScanRefer:   r.ScanObj.ID,
+// 			TargetRefer: r.ScanObj.TargetRefer,
+// 		}
+// 		obj.Create()
+// 	}
+// }
+
+// func (r *Runner) ImportTech(src string) {
+// 	if !utils.FileExists(src) {
+// 		utils.ErrorF("file not found: %v", src)
+// 		return
+// 	}
+
+// 	content := utils.ReadingLines(src)
+// 	for _, line := range content {
+// 		if !strings.Contains(line, ";;") {
+// 			utils.ErrorF("Invalid format: %v", line)
+// 			continue
+// 		}
+
+// 		// data should be domain|%v;;techs|%v
+// 		domain := strings.TrimPrefix(strings.Split(line, ";;")[0], "domain|")
+// 		techs := strings.TrimPrefix(strings.Split(line, ";;")[1], "techs|")
+// 		if strings.TrimSpace(techs) == "" {
+// 			utils.ErrorF("Invalid format: %v", line)
+// 			continue
+// 		}
+
+// 		obj := database.Asset{
+// 			AssetValue:  domain,
+// 			Technology:  techs,
+// 			IsAlive:     true,
+// 			ScanRefer:   r.ScanObj.ID,
+// 			TargetRefer: r.ScanObj.TargetRefer,
+// 		}
+
+// 		obj.UpdateTech()
+// 	}
+// }
+
+// func (r *Runner) ImportHTTPJson(src string) {
+// 	if !utils.FileExists(src) {
+// 		utils.ErrorF("file not found: %v", src)
+// 		return
+// 	}
+
+// 	content := utils.ReadingLines(src)
+
+// 	baseResult, _ := filepath.Abs(src)
+// 	baseResult = path.Dir(baseResult)
+// 	utils.DebugF("Set Base Dir for content: %v", baseResult)
+
+// 	for _, line := range content {
+// 		if strings.TrimSpace(line) == "" {
+// 			continue
+// 		}
+// 		jsonParsed, err := gabs.ParseJSON([]byte(line))
+// 		if err != nil {
+// 			continue
+// 		}
+
+// 		//  {"url":"https://sso-na.tesla.com","title":"Blank Title","checksum":"527ef0b39a78caf74f54ca5b2ffb59bcfe688685","content_file":"overview/contents/https___sso-na.tesla.com.txt","status":"302","time":"0.046268348","length":"90","redirect":"https://teamchatgl.tesla.com/"}
+// 		URL := jsonParsed.S("url").Data().(string)
+// 		title := jsonParsed.S("title").Data().(string)
+// 		checksum := jsonParsed.S("checksum").Data().(string)
+// 		contentPath := jsonParsed.S("content_file").Data().(string)
+// 		status := jsonParsed.S("status").Data().(string)
+// 		length := jsonParsed.S("length").Data().(string)
+// 		redirect := jsonParsed.S("redirect").Data().(string)
+
+// 		if !utils.FileExists(contentPath) {
+// 			contentPath = path.Join(baseResult, contentPath)
+// 		}
+
+// 		// base64 content
+// 		data := "No-Content"
+// 		if !strings.Contains(contentPath, "No-Content") {
+// 			//utils.DebugF("Reading HTML content: %v", contentPath)
+// 			data = utils.ImageAsBase64(contentPath)
+// 		}
+
+// 		obj := database.HTTP{
+// 			URL:           URL,
+// 			Title:         title,
+// 			Checksum:      checksum,
+// 			StatusCode:    cast.ToInt(status),
+// 			ContentLength: cast.ToInt(length),
+// 			HTTPContent:   data,
+// 			Redirect:      redirect,
+// 			ScanRefer:     r.ScanObj.ID,
+// 			TargetRefer:   r.ScanObj.TargetRefer,
+// 		}
+// 		obj.CreateHTTP()
+// 	}
+// }
+
+// func (r *Runner) ImportScreenShotJson(src string) {
+// 	if !utils.FileExists(src) {
+// 		utils.ErrorF("file not found: %v", src)
+// 		return
+// 	}
+
+// 	baseResult, _ := filepath.Abs(src)
+// 	baseResult = path.Dir(baseResult)
+// 	content := utils.ReadingLines(src)
+// 	utils.DebugF("Set Base Dir for screenshot: %v", baseResult)
+
+// 	for _, line := range content {
+// 		if strings.TrimSpace(line) == "" {
+// 			continue
+// 		}
+// 		jsonParsed, err := gabs.ParseJSON([]byte(line))
+// 		if err != nil {
+// 			continue
+// 		}
+
+// 		URL := jsonParsed.S("url").Data().(string)
+// 		imgPath := jsonParsed.S("image").Data().(string)
+// 		tech := jsonParsed.S("tech").Data().(string)
+// 		if tech != "" {
+// 			//domain, _ := utils.GetDomain(URL)
+// 			utils.DebugF("more tech: %v", tech)
+// 			//database.NewTech(wsObj, domain, tech)
+// 		}
+
+// 		if !utils.FileExists(imgPath) {
+// 			imgPath = path.Join(baseResult, imgPath)
+// 		}
+// 		imgData := utils.ImageAsBase64(imgPath)
+
+// 		//database.NewScreenshot(wsObj, URL, data)
+
+// 		obj := database.HTTP{
+// 			URL: URL,
+// 			//Title:          tittle,
+// 			//Checksum:       "",
+// 			//StatusCode:     0,
+// 			//ContentLength:  0,
+// 			//HTTPContent:    "",
+// 			ScreenShotData: imgData,
+// 			ScanRefer:      r.ScanObj.ID,
+// 			TargetRefer:    r.ScanObj.TargetRefer,
+// 		}
+// 		obj.CreateScreenShot()
+// 	}
+// }
+
+// type PortObj struct {
+// 	Protocol string
+// 	PortID   string
+// 	State    string
+// 	Service  struct {
+// 		Name    string
+// 		Product string
+// 		Cpe     string
+// 	}
+// 	Script struct {
+// 		ID     string
+// 		Output string
+// 	}
+// }
+
+// func (r *Runner) ImportPortJson(src string) {
+// 	if !utils.FileExists(src) {
+// 		utils.ErrorF("file not found: %v", src)
+// 		return
+// 	}
+// 	content := utils.ReadingLines(src)
+
+// 	for _, line := range content {
+// 		if strings.TrimSpace(line) == "" {
+// 			continue
+// 		}
+// 		jsonParsed, err := gabs.ParseJSON([]byte(line))
+// 		if err != nil {
+// 			continue
+// 		}
+
+// 		var portObj []PortObj
+
+// 		ipAddress := jsonParsed.S("IPAddress").Data().(string)
+// 		rawPorts := jsonParsed.S("Ports").Bytes()
+// 		err = jsoniter.Unmarshal(rawPorts, &portObj)
+// 		if err != nil || len(portObj) == 0 {
+// 			continue
+// 		}
+
+// 		var ports []string
+// 		for _, p := range portObj {
+// 			info := fmt.Sprintf("%v/%v/%v", p.PortID, p.Protocol, p.Service.Product)
+// 			ports = append(ports, strings.Trim(info, "/"))
+// 		}
+// 		if len(ports) == 0 {
+// 			continue
+// 		}
+
+// 		obj := database.Dns{
+// 			DnsValue:    ipAddress,
+// 			DnsType:     "A",
+// 			Ports:       strings.Join(ports, ","),
+// 			ScanRefer:   r.ScanObj.ID,
+// 			TargetRefer: r.ScanObj.TargetRefer,
+// 		}
+// 		obj.UpdatePort()
+// 	}
+// }
+
+// // ImportJaelesVulnJson import new asset to DB
+// func (r *Runner) ImportJaelesVulnJson(src string) {
+// 	if !utils.FileExists(src) {
+// 		utils.ErrorF("file not found: %v", src)
+// 		return
+// 	}
+// 	content := utils.ReadingLines(src)
+// 	baseDir := path.Dir(src)
+// 	for _, line := range content {
+// 		if strings.TrimSpace(line) == "" {
+// 			continue
+// 		}
+
+// 		jsonParsed, err := gabs.ParseJSON([]byte(line))
+// 		if err != nil {
+// 			utils.ErrorF("Error parse JSON Data")
+// 			continue
+// 		}
+// 		raw := jsonParsed.S("OutputFile").Data().(string)
+// 		reportPath := raw
+// 		if !utils.FileExists(reportPath) {
+// 			reportPath = path.Join(baseDir, raw)
+// 			if !utils.FileExists(reportPath) {
+// 				reportPath = path.Join(path.Dir(baseDir), raw)
+// 			}
+// 		}
+// 		vulnContent := utils.GetFileContent(reportPath)
+
+// 		// parse Data as JSON
+// 		jsonParsed, err = gabs.ParseJSON([]byte(vulnContent))
+// 		if err != nil {
+// 			utils.ErrorF("Error parse JSON Data")
+// 			continue
+// 		}
+
+// 		obj := database.Vulnerability{
+// 			URL:                jsonParsed.S("URL").Data().(string),
+// 			VulnRequest:        jsonParsed.S("Req").Data().(string),
+// 			VulnResponse:       jsonParsed.S("Res").Data().(string),
+// 			DetectionString:    jsonParsed.S("DetectionString").Data().(string),
+// 			VulnerabilityTitle: jsonParsed.S("SignName").Data().(string),
+// 			SignatureID:        jsonParsed.S("SignID").Data().(string),
+// 			Confidence:         jsonParsed.S("Confidence").Data().(string),
+// 			Severity:           jsonParsed.S("Risk").Data().(string),
+// 			Source:             "Jaeles",
+// 			//VulnChecksum:       utils.GenHash(vulnData),
+// 			//PluginScan: SelectScanID(workspace, pluginName, scanID),
+// 			//Target:     SelectScanByWS(workspace),
+// 			//Asset:      SelectAssetByData(domain),
+// 			ScanRefer:   r.ScanObj.ID,
+// 			TargetRefer: r.ScanObj.TargetRefer,
+// 		}
+// 		obj.Create()
+// 	}
+// }
+
+// // ImportNucleiVulnJson import new asset to DB
+// func (r *Runner) ImportNucleiVulnJson(src string) {
+// 	if !utils.FileExists(src) {
+// 		utils.ErrorF("file not found: %v", src)
+// 		return
+// 	}
+// 	content := utils.ReadingLines(src)
+// 	for _, line := range content {
+// 		if strings.TrimSpace(line) == "" {
+// 			continue
+// 		}
+// 		jsonParsed, err := gabs.ParseJSON([]byte(line))
+// 		if err != nil {
+// 			utils.ErrorF("Error parse JSON Data")
+// 			continue
+// 		}
+
+// 		obj := database.Vulnerability{
+// 			URL:             jsonParsed.S("host").Data().(string),
+// 			VulnRequest:     utils.Base64Encode(jsonParsed.S("request").Data().(string)),
+// 			VulnResponse:    utils.Base64Encode(jsonParsed.S("response").Data().(string)),
+// 			DetectionString: jsonParsed.S("matched").Data().(string),
+
+// 			SignatureID:        jsonParsed.S("templateID").Data().(string),
+// 			Confidence:         "Tentative",
+// 			VulnerabilityTitle: jsonParsed.S("info", "name").Data().(string),
+// 			Severity:           jsonParsed.S("info", "severity").Data().(string),
+// 			Source:             "Nuclei",
+// 			TargetRefer:        r.ScanObj.TargetRefer,
+
+// 			ScanRefer: r.ScanObj.ID,
+// 		}
+// 		obj.Create()
+// 	}
+// }
+
+// // ImportDirectoryJson import new asset to DB
+// func (r *Runner) ImportDirectoryJson(src string) {
+// 	if !utils.FileExists(src) {
+// 		utils.ErrorF("file not found: %v", src)
+// 		return
+// 	}
+// 	content := utils.ReadingLines(src)
+// 	for _, line := range content {
+// 		if strings.TrimSpace(line) == "" || !strings.Contains(line, "url") {
+// 			continue
+// 		}
+
+// 		jsonParsed, err := gabs.ParseJSON([]byte(line))
+// 		if err != nil {
+// 			utils.ErrorF("Error parse JSON Data")
+// 			continue
+// 		}
+
+// 		// some parser here
+// 		URL := jsonParsed.S("url").Data().(string)
+// 		redirect, ok := jsonParsed.S("redirectlocation").Data().(string)
+// 		if ok {
+// 			redirect = ""
+// 		}
+
+// 		obj := database.Directory{
+// 			URL:           URL,
+// 			Status:        cast.ToInt(jsonParsed.S("status").Data()),
+// 			ContentLength: cast.ToInt(jsonParsed.S("length").Data()),
+// 			Words:         cast.ToInt(jsonParsed.S("words").Data()),
+// 			RedirectURL:   redirect,
+// 			ScanRefer:     r.ScanObj.ID,
+// 			TargetRefer:   r.ScanObj.TargetRefer,
+// 		}
+// 		obj.Create()
+// 	}
+// }
+
+//
+//// UpdateJSONDns update dns in db
+//func UpdateJSONDns(src string, options libs.Options) {
+//	content := utils.ReadingLines(src)
+//	if len(content) == 0 {
+//		utils.ErrorF("File not found: %v", src)
+//		return
+//	}
+//
+//	utils.DebugF("Update JSON DNS: %v", src)
+//	//wsObj := database.SelectScanByWS(options.Scan.ROptions["Workspace"])
+//	target := options.Scan.ROptions["Workspace"]
+//
+//	for _, line := range content {
+//		jsonParsed, err := gabs.ParseJSON([]byte(line))
+//		if err != nil {
+//			continue
+//		}
+//
+//		domain, ok := jsonParsed.S("host").Data().(string)
+//		if !ok {
+//			continue
+//		}
+//
+//		// filtered some unrelated domain
+//		if !strings.HasSuffix(domain, fmt.Sprintf(".%s", target)) {
+//			if domain != target {
+//				continue
+//			}
+//		}
+//
+//		data := "N/A"
+//		isError := jsonParsed.S("status_code").Data().(string)
+//		if isError == "NXDOMAIN" {
+//			//database.NewAssetWithDns(wsObj, domain, data)
+//			continue
+//		}
+//		var results []string
+//		a := jsonParsed.S("a")
+//		if a != nil {
+//			for _, record := range a.Children() {
+//				data := fmt.Sprintf("A/%s", cast.ToString(record.Data()))
+//				results = append(results, data)
+//			}
+//		}
+//
+//		cname := jsonParsed.S("cname")
+//		if cname != nil {
+//			for _, record := range cname.Children() {
+//				data := fmt.Sprintf("CNAME/%s", cast.ToString(record.Data()))
+//				results = append(results, data)
+//			}
+//		}
+//
+//		mx := jsonParsed.S("mx")
+//		if mx != nil {
+//			for _, record := range mx.Children() {
+//				data := fmt.Sprintf("MX/%s", cast.ToString(record.Data()))
+//				results = append(results, data)
+//			}
+//		}
+//
+//		ns := jsonParsed.S("ns")
+//		if ns != nil {
+//			for _, record := range ns.Children() {
+//				data := fmt.Sprintf("NS/%s", cast.ToString(record.Data()))
+//				results = append(results, data)
+//			}
+//		}
+//		results = funk.UniqString(results)
+//		data = strings.Join(results, ";;")
+//		//database.NewAssetWithDns(wsObj, domain, data)
+//	}
+//}

core/import_mics.go

@@ -0,0 +1,238 @@
+package core
+
+// "github.com/j3ssie/osmedeus/database"
+
+// func (r *Runner) StartScanNoti() error {
+// 	data, err := jsoniter.Marshal(&r.ScanObj)
+// 	if err != nil {
+// 		return fmt.Errorf("err marshal object: %v", err)
+// 	}
+// 	noti := database.Notification{
+// 		NotificationType:   "start",
+// 		NotificationSource: "scan",
+// 		NewData:            datatypes.JSON(data),
+// 		ObjRefer:           r.ScanObj.ID,
+// 		ScanRefer:          r.ScanObj.ID,
+// 		TargetRefer:        r.TargetObj.ID,
+// 	}
+// 	return noti.Create()
+// }
+
+// func (r *Runner) ScanDoneNoti() error {
+// 	data, err := jsoniter.Marshal(&r.ScanObj)
+// 	if err != nil {
+// 		return fmt.Errorf("err marshal object: %v", err)
+// 	}
+// 	noti := database.Notification{
+// 		NotificationType:   "done",
+// 		NotificationSource: "scan",
+// 		NewData:            datatypes.JSON(data),
+// 		ObjRefer:           r.ScanObj.ID,
+// 		ScanRefer:          r.ScanObj.ID,
+// 		TargetRefer:        r.TargetObj.ID,
+// 	}
+// 	return noti.Create()
+// }
+
+// // ImportLinks import new link to DB
+// func (r *Runner) ImportLinks(src string) {
+// 	if !utils.FileExists(src) {
+// 		utils.ErrorF("file not found: %v", src)
+// 		return
+// 	}
+
+// 	content := utils.ReadingLines(src)
+// 	for _, line := range content {
+// 		if strings.TrimSpace(line) == "" {
+// 			continue
+// 		}
+// 		// {"input":"http://academy.live-test.redacted.io","source":"body","type":"url","output":"http://academy.live-test.redacted.io","status":307,"length":7}
+// 		jsonParsed, err := gabs.ParseJSON([]byte(line))
+// 		if err != nil {
+// 			utils.ErrorF("Error parse JSON Data")
+// 			continue
+// 		}
+
+// 		obj := database.Link{
+// 			LinkValue:   jsonParsed.S("output").Data().(string),
+// 			LinkSource:  jsonParsed.S("source").Data().(string),
+// 			URL:         jsonParsed.S("input").Data().(string),
+// 			LinkType:    jsonParsed.S("type").Data().(string),
+// 			ScanRefer:   r.ScanObj.ID,
+// 			TargetRefer: r.TargetObj.ID,
+// 		}
+// 		obj.Create()
+// 	}
+// }
+
+// func (r *Runner) ImportArchive(src string) {
+// 	if !utils.FileExists(src) {
+// 		utils.ErrorF("file not found: %v", src)
+// 		return
+// 	}
+
+// 	content := utils.ReadingLines(src)
+// 	for _, line := range content {
+// 		if strings.TrimSpace(line) == "" {
+// 			continue
+// 		}
+
+// 		// ignore root domain
+// 		u, err := url.Parse(line)
+// 		if err != nil || u.Path == "" || u.Path == "/" {
+// 			continue
+// 		}
+
+// 		obj := database.Archive{
+// 			ArchiveValue:    line,
+// 			ArchiveChecksum: utils.GenHash(line),
+// 			ScanRefer:       r.ScanObj.ID,
+// 			TargetRefer:     r.TargetObj.ID,
+// 		}
+// 		obj.Create()
+// 	}
+// }
+
+// func (r *Runner) ImportIPRange(src string) {
+// 	if !utils.FileExists(src) {
+// 		utils.ErrorF("file not found: %v", src)
+// 		return
+// 	}
+
+// 	content := utils.ReadingLines(src)
+// 	for _, line := range content {
+// 		if strings.TrimSpace(line) == "" {
+// 			continue
+// 		}
+// 		jsonParsed, err := gabs.ParseJSON([]byte(line))
+// 		if err != nil {
+// 			utils.ErrorF("Error parse JSON Data")
+// 			continue
+// 		}
+
+// 		utils.DebugF("Processing: %v", line)
+// 		obj := database.IPRange{
+// 			ASNumber:    cast.ToString(jsonParsed.S("Number").Data()),
+// 			Country:     cast.ToString(jsonParsed.S("CountryCode").Data()),
+// 			Value:       cast.ToString(jsonParsed.S("CIDR").Data()),
+// 			Info:        cast.ToString(jsonParsed.S("Description").Data()),
+// 			Amount:      cast.ToUint(jsonParsed.S("Count").Data()),
+// 			ScanRefer:   r.ScanObj.ID,
+// 			TargetRefer: r.TargetObj.ID,
+// 		}
+// 		obj.Create()
+// 	}
+// }
+
+// func (r *Runner) ImportCert(src string) {
+// 	if !utils.FileExists(src) {
+// 		utils.ErrorF("file not found: %v", src)
+// 		return
+// 	}
+
+// 	content := utils.ReadingLines(src)
+// 	for _, line := range content {
+// 		if strings.TrimSpace(line) == "" {
+// 			continue
+// 		}
+// 		jsonParsed, err := gabs.ParseJSON([]byte(line))
+// 		if err != nil {
+// 			utils.ErrorF("Error parse JSON Data")
+// 			continue
+// 		}
+
+// 		domain := cast.ToString(jsonParsed.S("Domain").Data())
+// 		isWildCard := false
+// 		if strings.Contains(domain, "*.") {
+// 			isWildCard = true
+// 			domain = strings.TrimLeft(domain, "*.")
+// 		}
+
+// 		obj := database.CertInfo{
+// 			Domain:      domain,
+// 			CertInfo:    cast.ToString(jsonParsed.S("CertInfo").Data()),
+// 			OrgInfo:     cast.ToString(jsonParsed.S("OrgInfo").Data()),
+// 			IsWildcard:  isWildCard,
+// 			TargetRefer: r.TargetObj.ID,
+// 		}
+// 		obj.Create()
+// 	}
+// }
+
+// func (r *Runner) ImportCred(src string) {
+// 	if !utils.FileExists(src) {
+// 		utils.ErrorF("file not found: %v", src)
+// 		return
+// 	}
+
+// 	content := utils.ReadingLines(src)
+// 	for _, line := range content {
+// 		if strings.TrimSpace(line) == "" {
+// 			continue
+// 		}
+
+// 		jsonParsed, err := gabs.ParseJSON([]byte(line))
+// 		if err != nil {
+// 			utils.ErrorF("Error parse JSON Data")
+// 			continue
+// 		}
+
+// 		email := cast.ToString(jsonParsed.S("email").Data())
+// 		credID := cast.ToString(jsonParsed.S("id").Data())
+// 		username := cast.ToString(jsonParsed.S("username").Data())
+// 		password := cast.ToString(jsonParsed.S("password").Data())
+// 		hashedPassword := cast.ToString(jsonParsed.S("hashed_password").Data())
+// 		name := cast.ToString(jsonParsed.S("name").Data())
+// 		ipAddress := cast.ToString(jsonParsed.S("ip_address").Data())
+// 		phone := cast.ToString(jsonParsed.S("phone").Data())
+// 		source := cast.ToString(jsonParsed.S("database_name").Data())
+
+// 		obj := database.Credential{
+// 			CredID:         credID,
+// 			Email:          email,
+// 			Username:       username,
+// 			Password:       password,
+// 			HashedPassword: hashedPassword,
+// 			Name:           name,
+// 			Phone:          phone,
+// 			IPAddress:      ipAddress,
+// 			Source:         source,
+// 			TargetRefer:    r.TargetObj.ID,
+// 			ScanRefer:      r.ScanObj.ID,
+// 		}
+// 		obj.Create()
+// 	}
+// }
+
+// func (r *Runner) ImportCloudBrute(src string) {
+// 	if !utils.FileExists(src) {
+// 		utils.ErrorF("file not found: %v", src)
+// 		return
+// 	}
+
+// 	content := utils.ReadingLines(src)
+// 	for _, line := range content {
+// 		if strings.TrimSpace(line) == "" {
+// 			continue
+// 		}
+
+// 		if !strings.Contains(line, " - ") {
+// 			continue
+// 		}
+
+// 		cloudDomain := strings.Split(line, " - ")[1]
+// 		status := strings.Split(line, " - ")[0]
+// 		if strings.Contains(status, ": ") {
+// 			status = strings.Split(strings.Split(line, " - ")[0], ": ")[1]
+// 		}
+
+// 		obj := database.CloudBrute{
+// 			Status:      status,
+// 			CloudDomain: cloudDomain,
+// 			RawData:     line,
+// 			ScanRefer:   r.ScanObj.ID,
+// 			TargetRefer: r.TargetObj.ID,
+// 		}
+// 		obj.Create()
+// 	}
+// }

core/markdown.go

@@ -0,0 +1,256 @@
+package core
+
+import (
+	"path"
+
+	"fmt"
+	"os"
+	"regexp"
+	"strings"
+
+	"github.com/fatih/color"
+	"github.com/j3ssie/osmedeus/libs"
+	"github.com/j3ssie/osmedeus/utils"
+	"github.com/spf13/cast"
+
+	"github.com/gomarkdown/markdown"
+	"github.com/gomarkdown/markdown/html"
+	"github.com/gomarkdown/markdown/parser"
+)
+
+func (r *Runner) GenMarkdownReport(markdownFile string, outputHTML string) {
+	utils.DebugF("Reading markdown report from: %v", markdownFile)
+
+	// get the markdown template content
+	mdContent := utils.GetFileContent(markdownFile)
+	mdContent = ResolveData(mdContent, r.Target)
+
+	// replace all the <scanInfo /> tag
+	mdContent = r.ResolveScanInfoTag(mdContent)
+	// utils.DebugF("ResolveScanInfoTag:\n%v", mdContent)
+
+	// replace all the <reports /> tag
+	mdContent = r.ResolveReportsTag(mdContent)
+
+	// replace all the <content /> tag
+	mdContent = r.ResolveContentTag(mdContent)
+	// fmt.Println("mdContent", mdContent)
+
+	// generating the markdown file first
+	outputMD := strings.Replace(outputHTML, ".html", ".md", -1)
+	utils.WriteToFile(outputMD, mdContent)
+
+	utils.InforF("Generate markdown report: %v", outputMD)
+	utils.InforF("Generate HTML report: %v", outputHTML)
+	// finally convert to HTML
+	MarkDownToHTML(r.Opt, r.Input, outputMD, outputHTML)
+}
+
+func (r *Runner) ResolveScanInfoTag(rawMarkdown string) string {
+	re := regexp.MustCompile(`<scanInfo\s*/>`)
+	match := re.FindString(rawMarkdown)
+	if len(match) > 1 {
+		utils.DebugF("Replace scanInfo tag: %v", match)
+		scanInfo := fmt.Sprintf(`
+| <!-- -->       | <!-- -->    |
+|----------------|-------------|
+| Target         | **:target**     |
+| Running Time   | **:runningTime**         |
+| Workflow       | **:workflow**         |
+| Status         | **:status**         |
+| Statistics     | :statistics         |
+`)
+
+		status := "done"
+		if r.ScanObj.IsRunning {
+			status = "running"
+		}
+
+		statistics := fmt.Sprintf("`assets/%v`, `dns/%v`, `vulnerability/%v`, ", r.TargetObj.TotalAssets, r.TargetObj.TotalDns, r.TargetObj.TotalVulnerability)
+		replacements := map[string]string{
+			":target":      r.ScanObj.InputName,
+			":runningTime": cast.ToString(int(r.RunningTime)/3600) + " hours",
+			":workflow":    r.ScanObj.TaskName,
+			":statistics":  statistics,
+			":status":      status,
+		}
+		// generate the statistics info
+		for oldStr, newStr := range replacements {
+			scanInfo = strings.ReplaceAll(scanInfo, oldStr, newStr)
+		}
+
+		return strings.Replace(rawMarkdown, match, scanInfo, -1)
+	}
+	utils.DebugF("No scanInfo tag found")
+	return rawMarkdown
+}
+
+func (r *Runner) ResolveContentTag(rawData string) string {
+	finalMarkdown := rawData
+	// finding all the content tags and replace it with the content
+	re := regexp.MustCompile(`<content[^>]*>`)
+	matchs := re.FindAllString(rawData, -1)
+	for _, contentTag := range matchs {
+		utils.DebugF("Replace content tag: %v", color.GreenString(contentTag))
+		content := r.ResolveContentSrc(contentTag)
+		finalMarkdown = strings.Replace(finalMarkdown, contentTag, content, -1)
+	}
+
+	return finalMarkdown
+}
+
+func (r *Runner) ResolveContentSrc(tag string) string {
+	re := regexp.MustCompile(`src=\"(\S+)\"`)
+	match := re.FindStringSubmatch(tag)
+	if len(match) > 1 {
+		fileContent := utils.GetFileContent(match[1])
+		utils.DebugF("Replace content src: %v", color.GreenString(match[1]))
+
+		if strings.Contains(tag, "expand=true") {
+			return "```\n" + fileContent + "```"
+		}
+
+		if strings.Contains(tag, "shorten=true") || len(fileContent) > r.Opt.MDCodeBlockLimit {
+			return extendTag(fileContent)
+		}
+
+		return "```\n" + fileContent + "```"
+	}
+	return ""
+}
+
+func extendTag(str string) string {
+	data := "<details>\n<summary>Click to Expand</summary>\n\n" + "<pre>\n" + str + "\n</pre>" + "\n</details>"
+	return data
+}
+
+func (r *Runner) ResolveReportsTag(rawMarkDown string) string {
+	// rawData := utils.GetFileContent(markdownFile)
+	finalMarkdown := rawMarkDown
+	// finding all the reports tags
+	re := regexp.MustCompile(`<reports\s*/>`)
+	matchs := re.FindAllString(rawMarkDown, -1)
+	if len(matchs) == 0 {
+		utils.DebugF("No reports tag found")
+		return rawMarkDown
+	}
+
+	for _, reportTag := range matchs {
+		utils.DebugF("Replace content tag: %v", reportTag)
+		mdContent := ""
+
+		for _, report := range r.TargetObj.Reports {
+			// add the link if report file is HTML file
+			if report.ReportType == "html" {
+				mdContent += fmt.Sprintf("### %s -- [%s](%s) \n\n", report.Module, report.ReportName, report.ReportPath)
+				mdContent += "\n***\n"
+				continue
+			}
+			// add the full content if report file is a text file
+			mdContent += fmt.Sprintf("### %s -- *%s* \n\n", report.Module, report.ReportName)
+
+			fileContent := utils.GetFileContent(report.ReportPath)
+			if len(fileContent) > r.Opt.MDCodeBlockLimit {
+				mdContent += extendTag(fileContent)
+			} else {
+				mdContent += "```\n"
+				mdContent += fileContent
+				mdContent += "\n```\n"
+			}
+			mdContent += "\n***\n\n"
+		}
+
+		finalMarkdown = strings.Replace(finalMarkdown, reportTag, mdContent, -1)
+	}
+
+	return finalMarkdown
+}
+
+func MarkDownToHTML(options libs.Options, target string, markdownFile string, outputFile string) error {
+	css := path.Join(options.Env.DataFolder, "markdown/style.css")
+
+	var input []byte
+	var err error
+
+	if input, err = os.ReadFile(markdownFile); err != nil {
+		utils.ErrorF("Error reading %s: %v", markdownFile, err)
+		return err
+	}
+
+	// set up options
+	var extensions = parser.NoIntraEmphasis |
+		parser.Tables |
+		parser.FencedCode |
+		parser.Autolink |
+		parser.Strikethrough |
+		parser.SpaceHeadings
+
+	var renderer markdown.Renderer
+	// render the data into HTML
+	var htmlFlags html.Flags
+
+	htmlFlags |= html.Smartypants
+	htmlFlags |= html.UseXHTML
+	htmlFlags |= html.CompletePage
+	htmlFlags |= html.SmartypantsLatexDashes
+	htmlFlags |= html.SmartypantsFractions
+
+	params := html.RendererOptions{
+		Flags: htmlFlags,
+		CSS:   css,
+	}
+	renderer = html.NewRenderer(params)
+
+	// parse and render
+	var output []byte
+	parser := parser.NewWithExtensions(extensions)
+	// @NOTE: beware of XSS as I assume you will trust the markdown content you generate
+	output = markdown.ToHTML(input, parser, renderer)
+	// html := bluemonday.UGCPolicy().SanitizeBytes(output) // skip the sanitization as we prefer more beautiful output
+
+	cssContent, err := os.ReadFile(css)
+	if err != nil {
+		return err
+	}
+
+	finalHTML := fmt.Sprintf(`
+<html>
+	<head>
+	<meta charset="UTF-8">
+	<meta name="viewport" content="width=device-width">
+	<link rel="icon" type="image/svg+xml" href="https://www.osmedeus.org/favicon.png">
+	<meta name="generator" content="Generated by https://github.com/gomarkdown/markdown">
+	<meta name="description" content="A Workflow Engine for Offensive Security">
+	<title>Osmedeus Executive Summary - %s</title>
+	<meta property="og:title" content="Osmedeus Next Generation">
+	<meta property="og:type" content="website">
+	<meta property="og:url" content="https://www.osmedeus.org">
+	<meta property="og:image" content="https://raw.githubusercontent.com/osmedeus/assets/main/banner.png">
+	<meta property="og:description" content="A Workflow Engine for Offensive Security">
+	<meta name="twitter:card" content="">
+	<meta name="twitter:site" content="@OsmedeusEngine">
+	<meta name="twitter:creator" content="@OsmedeusEngine">
+	<style>%s</style>
+	</head>
+
+<body>
+%s
+</body>
+</html>
+	`, target, string(cssContent), string(output))
+
+	// output the result
+	var out *os.File
+	if out, err = os.Create(outputFile); err != nil {
+		utils.ErrorF("Error creating %s: %v", outputFile, err)
+		return err
+	}
+	defer out.Close()
+
+	if _, err = out.WriteString(finalHTML); err != nil {
+		utils.ErrorF("Error writing output: %v", err)
+		return err
+	}
+
+	return nil
+}

core/mode_test.go

@@ -0,0 +1,24 @@
+package core
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/j3ssie/osmedeus/libs"
+)
+
+func TestListMode(t *testing.T) {
+	var options libs.Options
+	options.Env.WorkFlowsFolder = "~/go/src/github.com/j3ssie/osmedeus/workflow/"
+	result := ListFlow(options)
+	fmt.Println(result)
+	if len(result) == 0 {
+		t.Errorf("Error ListMode")
+	}
+
+	selectedMode := SelectFlow("general", options)
+	fmt.Println(selectedMode)
+	if len(selectedMode) == 0 {
+		t.Errorf("Error selectedMode")
+	}
+}

core/module.go

@@ -0,0 +1,370 @@
+package core
+
+import (
+	"context"
+	"fmt"
+	"path"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/spf13/cast"
+
+	"github.com/fatih/color"
+	"github.com/j3ssie/osmedeus/execution"
+	"github.com/j3ssie/osmedeus/libs"
+	"github.com/j3ssie/osmedeus/utils"
+	"github.com/panjf2000/ants"
+)
+
+// RunModule run the module
+func (r *Runner) RunModule(module libs.Module) {
+	// get reports path
+	module = ResolveReports(module, r.Params)
+
+	// check if resume enable or not
+	if (r.Opt.Resume || module.Resume) && !module.Forced {
+		if CheckResume(module) {
+			utils.BlockF(module.Name, "Resume detected")
+			return
+		}
+	}
+
+	r.CurrentModule = module.Name
+	timeStart := time.Now()
+	utils.BlockF("Module-Started", fmt.Sprintf("%v - %v", module.Name, module.Desc))
+
+	// create report record first because I don't want to wait for them to show up in UI until the module done
+	r.DBNewReports(module)
+
+	// pre-run
+	if len(module.PreRun) > 0 && r.Opt.NoPreRun == false {
+		utils.InforF("Running prepare scripts for module %v", color.CyanString(module.Name))
+		r.RunScripts(module.PreRun)
+	}
+
+	// main part
+	// utils.BlockF(module.Name, "Begin executing primary tasks")
+	err := r.RunSteps(module.Steps)
+	if err != nil {
+		utils.BadBlockF(module.Name, fmt.Sprintf("got exit call"))
+	}
+
+	// post-run
+	if len(module.PostRun) > 0 && r.Opt.NoPostRun == false {
+		utils.InforF("Running conclude scripts for module %v", color.CyanString(module.Name))
+		r.RunScripts(module.PostRun)
+	}
+
+	// print the reports file
+	utils.PrintLine()
+	printReports(module)
+
+	// estimate time
+	elapsedTime := time.Since(timeStart).Seconds()
+	utils.BlockF("Module-Ended", fmt.Sprintf("Elapsed Time for the module %v in %v", color.HiCyanString(module.Name), color.HiMagentaString("%vs", elapsedTime)))
+	r.RunningTime += cast.ToInt(elapsedTime)
+	utils.PrintLine()
+	r.DBUpdateScan()
+}
+
+// RunScripts run list of scripts
+func (r *Runner) RunScripts(scripts []string) string {
+	if r.Opt.Timeout != "" {
+		timeout := utils.CalcTimeout(r.Opt.Timeout)
+		utils.DebugF("Run scripts with %v seconds timeout", timeout)
+		r.RunScriptsWithTimeOut(r.Opt.Timeout, scripts)
+		return ""
+	}
+
+	for _, script := range scripts {
+		outScript := r.RunScript(script)
+		if strings.Contains(outScript, "exit") {
+			return outScript
+		}
+	}
+	return ""
+}
+
+// RunScriptsWithTimeOut run list of scripts with timeout
+func (r *Runner) RunScriptsWithTimeOut(timeoutRaw string, scripts []string) string {
+	timeout := utils.CalcTimeout(timeoutRaw)
+	utils.DebugF("Run scripts with %v seconds timeout", timeout)
+
+	c := context.Background()
+	deadline := time.Now().Add(time.Duration(timeout) * time.Second)
+	c, cancel := context.WithDeadline(c, deadline)
+	defer cancel()
+
+	go func() {
+		for _, script := range scripts {
+			outScript := r.RunScript(script)
+			if strings.Contains(outScript, "exit") {
+				return
+			}
+		}
+		cancel()
+	}()
+
+	select {
+	case <-c.Done():
+		utils.DebugF("Scripts done")
+		return ""
+	case <-time.After(time.Duration(timeout) * time.Second):
+		utils.BadBlockF("timeout", fmt.Sprintf("Scripts got timeout after %v", color.HiMagentaString(timeoutRaw)))
+	}
+	return ""
+}
+
+// RunScript really run a script
+func (r *Runner) RunScript(script string) string {
+	return r.ExecScript(script)
+}
+
+// RunSteps run list of steps
+func (r *Runner) RunSteps(steps []libs.Step) error {
+	var stepOut string
+	for _, step := range steps {
+		r.DoneStep += 1
+
+		if step.Timeout != "" {
+			// timeout should be: 30, 30m, 1h
+			timeout := utils.CalcTimeout(step.Timeout)
+			if timeout != 0 {
+				stepOut, _ = r.RunStepWithTimeout(timeout, step)
+				if strings.Contains(stepOut, "exit") {
+					return fmt.Errorf("got exit call")
+				}
+				continue
+			}
+		}
+
+		stepOut, _ = r.RunStep(step)
+		if strings.Contains(stepOut, "exit") {
+			return fmt.Errorf("got an exit call")
+		}
+	}
+	return nil
+}
+
+// RunStepWithTimeout run step with timeout
+func (r *Runner) RunStepWithTimeout(timeout int, step libs.Step) (out string, err error) {
+	utils.DebugF("Run step with %v seconds timeout", timeout)
+	prefix := fmt.Sprintf("timeout -k 1m %vs ", timeout)
+
+	// prepare the os command with prefix timeout first
+	var preFixCommands []string
+	for _, command := range step.Commands {
+		preFixCommand := command
+		if !strings.Contains(command, "timeout") {
+			preFixCommand = prefix + command
+		}
+		preFixCommands = append(preFixCommands, preFixCommand)
+	}
+	step.Commands = preFixCommands
+
+	// override global timeout
+	r.Opt.Timeout = step.Timeout
+	return r.RunStep(step)
+}
+
+func (r *Runner) RunStep(step libs.Step) (string, error) {
+	var output string
+	if step.Label != "" {
+		utils.BlockF("Step", fmt.Sprintf("Initiating Step %v", color.HiGreenString(step.Label)))
+	}
+
+	// checking required file
+	err := r.CheckRequired(step.Required)
+	if err != nil {
+		return output, fmt.Errorf("missing requirements")
+	}
+
+	// check conditions and run reverse step
+	err = r.CheckCondition(step.Conditions)
+	if err != nil {
+		if len(step.RCommands) == 0 && len(step.RScripts) == 0 {
+			return output, fmt.Errorf("conditions not met")
+		}
+
+		// run reverse commands
+		utils.InforF("Condition false, run the reverse commands")
+		if len(step.RCommands) > 0 {
+			r.RunCommands(step.RCommands, step.Std)
+		}
+		// run reverse scripts
+		if len(step.RScripts) > 0 {
+			output = r.RunScripts(step.RScripts)
+			if strings.Contains(output, "exit") {
+				return output, nil
+			}
+		}
+		return output, nil
+	}
+
+	// run the step in loop mode
+	if step.Source != "" {
+		return r.RunStepWithSource(step)
+	}
+	//
+
+	if len(step.Commands) > 0 {
+		r.RunCommands(step.Commands, step.Std)
+	}
+	if len(step.Scripts) > 0 {
+		output = r.RunScripts(step.Scripts)
+		if strings.Contains(output, "exit") {
+			return output, nil
+		}
+	}
+
+	// run ose here
+	if len(step.Ose) > 0 {
+		for _, ose := range step.Ose {
+			r.RunOse(ose)
+		}
+	}
+
+	// post scripts
+	if len(step.PConditions) > 0 || len(step.PScripts) > 0 {
+		err := r.CheckCondition(step.PConditions)
+		if err == nil {
+			if len(step.PScripts) > 0 {
+				r.RunScripts(step.PScripts)
+			}
+		}
+	}
+	return output, nil
+
+}
+
+// RunStepWithSource really run a step
+func (r *Runner) RunStepWithSource(step libs.Step) (out string, err error) {
+	////// Start to run step but in loop mode
+	utils.DebugF("Running the step using the source file: %v", step.Source)
+	data := utils.ReadingLines(step.Source)
+	if len(data) <= 0 {
+		return out, fmt.Errorf("missing source")
+	}
+	if step.Threads != "" {
+		step.Parallel = cast.ToInt(step.Threads)
+	}
+	if step.Parallel == 0 {
+		step.Parallel = 1
+	}
+
+	// prepare the data first
+	var newGeneratedSteps []libs.Step
+	for index, line := range data {
+		customParams := make(map[string]string)
+		customParams["line"] = line
+		customParams["line_id"] = fmt.Sprintf("%v-%v", path.Base(line), index)
+		customParams["_id_"] = fmt.Sprintf("%v", index)
+		customParams["_line_"] = execution.StripName(line)
+
+		// make completely new Step
+		localStep := libs.Step{}
+
+		for _, cmd := range step.Commands {
+			localStep.Commands = append(localStep.Commands, AltResolveVariable(cmd, customParams))
+		}
+		for _, cmd := range step.RCommands {
+			localStep.RCommands = append(localStep.RCommands, AltResolveVariable(cmd, customParams))
+		}
+
+		if len(step.Ose) > 0 {
+			for _, ose := range step.Ose {
+				localStep.Ose = append(localStep.Ose, AltResolveVariable(ose, customParams))
+			}
+		}
+
+		for _, script := range step.RScripts {
+			localStep.RScripts = append(localStep.RScripts, AltResolveVariable(script, customParams))
+		}
+
+		for _, script := range step.Scripts {
+			localStep.Scripts = append(localStep.Scripts, AltResolveVariable(script, customParams))
+		}
+
+		for _, script := range step.PConditions {
+			localStep.PConditions = append(localStep.PConditions, AltResolveVariable(script, customParams))
+		}
+		for _, script := range step.PScripts {
+			localStep.PScripts = append(localStep.PScripts, AltResolveVariable(script, customParams))
+		}
+
+		newGeneratedSteps = append(newGeneratedSteps, localStep)
+	}
+
+	// skip concurrency part
+	if step.Parallel == 1 {
+		for _, newGeneratedStep := range newGeneratedSteps {
+			out, err = r.RunStep(newGeneratedStep)
+			if err != nil {
+				continue
+			}
+		}
+	} else {
+		/////////////
+		// run multiple steps in concurrency mode
+
+		utils.DebugF("Running the step in parallel: %v", step.Parallel)
+		var wg sync.WaitGroup
+		p, _ := ants.NewPoolWithFunc(step.Parallel, func(i interface{}) {
+			r.startStepJob(i)
+			wg.Done()
+		}, ants.WithPreAlloc(true))
+		defer p.Release()
+
+		for _, newGeneratedStep := range newGeneratedSteps {
+			wg.Add(1)
+			err = p.Invoke(newGeneratedStep)
+			if err != nil {
+				utils.ErrorF("Error in parallel: %v", err)
+			}
+		}
+
+		wg.Wait()
+	}
+	return out, nil
+}
+
+func (r *Runner) startStepJob(j interface{}) {
+	localStep := j.(libs.Step)
+
+	err := r.CheckCondition(localStep.Conditions)
+
+	if err != nil {
+		// run reverse commands
+		if len(localStep.RCommands) > 0 {
+			r.RunCommands(localStep.RCommands, localStep.Std)
+		}
+		if len(localStep.RScripts) > 0 {
+			r.RunScripts(localStep.RScripts)
+		}
+	} else {
+		if len(localStep.Commands) > 0 {
+			r.RunCommands(localStep.Commands, localStep.Std)
+		}
+	}
+
+	if len(localStep.Ose) > 0 {
+		for _, ose := range localStep.Ose {
+			r.RunOse(ose)
+		}
+	}
+
+	if len(localStep.Scripts) > 0 {
+		r.RunScripts(localStep.Scripts)
+	}
+
+	// post scripts
+	if len(localStep.PConditions) > 0 || len(localStep.PScripts) > 0 {
+		err := r.CheckCondition(localStep.PConditions)
+		if err == nil {
+			if len(localStep.PScripts) > 0 {
+				r.RunScripts(localStep.PScripts)
+			}
+		}
+	}
+}

core/parse.go

@@ -0,0 +1,322 @@
+package core
+
+import (
+	"bytes"
+	"fmt"
+	"net/url"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"text/template"
+	"time"
+
+	"github.com/Jeffail/gabs/v2"
+	"github.com/fatih/color"
+	"github.com/flosch/pongo2/v6"
+	"github.com/spf13/cast"
+	"golang.org/x/net/publicsuffix"
+
+	"github.com/Shopify/yaml"
+	"github.com/j3ssie/osmedeus/libs"
+	"github.com/j3ssie/osmedeus/utils"
+)
+
+// ResolveData resolve template from signature file
+func ResolveData(format string, data map[string]string) string {
+	// for backward compatibility because new template using `{{variable}}` instead of `{{.variable}}`
+	if strings.Contains(format, "{{.") {
+		return OldResolveData(format, data)
+	}
+
+	variable := make(map[string]interface{})
+	for k, v := range data {
+		variable[k] = v
+	}
+	if tpl, err := pongo2.FromString(format); err == nil {
+		out, ok := tpl.Execute(variable)
+		if ok == nil {
+			return out
+		}
+		utils.ErrorF("Error when resolve template: %v", ok)
+	}
+	return format
+}
+
+// OldResolveData resolve template from signature file
+func OldResolveData(format string, data map[string]string) string {
+	t := template.Must(template.New("").Parse(format))
+
+	buf := &bytes.Buffer{}
+	err := t.Execute(buf, data)
+	if err != nil {
+		utils.ErrorF("Error render: %v -- %v", format, err)
+		return format
+	}
+	return buf.String()
+}
+
+// ResolveSlice resolve template from signature file
+func ResolveSlice(slice []string, data map[string]string) (resolveSlice []string) {
+	for _, s := range slice {
+		resolveSlice = append(resolveSlice, ResolveData(s, data))
+	}
+	return resolveSlice
+}
+
+// AltResolveVariable just like ResolveVariable but looking for [[.var]]
+func AltResolveVariable(format string, data map[string]string) string {
+	t := template.Must(template.New("").Delims("[[", "]]").Parse(format))
+	buf := &bytes.Buffer{}
+	err := t.Execute(buf, data)
+	if err != nil {
+		return format
+	}
+	return buf.String()
+}
+
+// ParseFlow parse mode file
+func ParseFlow(flowFile string) (libs.Flow, error) {
+	utils.DebugF("Parsing workflow at: %v", color.HiGreenString(flowFile))
+	var flow libs.Flow
+	yamlFile, err := os.ReadFile(flowFile)
+	if err != nil {
+		utils.ErrorF("YAML parsing err: %v -- #%v ", flowFile, err)
+		return flow, err
+	}
+	err = yaml.Unmarshal(yamlFile, &flow)
+	if err != nil {
+		utils.ErrorF("Error unmarshal: %v -- %v", flowFile, err)
+		return flow, err
+	}
+
+	if flow.Usage != "" && strings.Contains(flow.Usage, "{{.this_file}}") {
+		flow.Usage = strings.ReplaceAll(flow.Usage, "{{.this_file}}", flowFile)
+	}
+	return flow, nil
+}
+
+// ParseModules parse module file
+func ParseModules(moduleFile string) (libs.Module, error) {
+	utils.DebugF("Parsing module at: %v", color.HiCyanString(moduleFile))
+
+	var module libs.Module
+
+	yamlFile, err := os.ReadFile(moduleFile)
+	if err != nil {
+		utils.ErrorF("YAML parsing err: %v -- #%v ", moduleFile, err)
+		return module, err
+	}
+	err = yaml.Unmarshal(yamlFile, &module)
+	if err != nil {
+		utils.ErrorF("Error unmarshal: %v -- %v", moduleFile, err)
+		return module, err
+	}
+	module.ModulePath = moduleFile
+	if module.Usage != "" && strings.Contains(module.Usage, "{{this_file}}") {
+		module.Usage = strings.ReplaceAll(module.Usage, "{{this_file}}", moduleFile)
+	}
+	return module, err
+}
+
+// ParseInputFormat format input
+func ParseInputFormat(raw string, options libs.Options) map[string]string {
+	target := make(map[string]string)
+	target["RawFormat"] = raw
+
+	jsonParsed, err := gabs.ParseJSON([]byte(raw))
+	if err != nil {
+		return target
+	}
+
+	// parse Target first if found one
+	rawURL, ok := jsonParsed.ChildrenMap()["Target"]
+	if !ok {
+		panic("missing Target in special input")
+	}
+
+	target = ParseInput(cast.ToString(rawURL.Data()), options)
+
+	// override the whole thing
+	for k, v := range jsonParsed.ChildrenMap() {
+		target[k] = fmt.Sprintf("%v", v.Data())
+	}
+
+	return target
+}
+
+// ParseInput parse input for routine
+func ParseInput(raw string, options libs.Options) map[string]string {
+	ROptions := ParseTarget(raw)
+	if options.EnableFormatInput {
+		// avoid the loophole
+		options.EnableFormatInput = false
+		ROptions = ParseInputFormat(raw, options)
+		return ROptions
+	}
+	// some data stuff
+	dir, err := os.Getwd()
+	if err == nil {
+		ROptions["CWD"] = dir
+	}
+
+	// default threads variables
+	ROptions["Threads"] = cast.ToString(options.Threads)
+	ROptions["threads"] = cast.ToString(options.Threads)
+	ROptions["thread"] = cast.ToString(options.Threads)
+	ROptions["baseThreads"] = cast.ToString(options.Threads)
+
+	ROptions["Version"] = libs.VERSION
+	ROptions["Bucket"] = options.Cdn.Bucket
+
+	ROptions["Today"] = time.Now().Format("2006-01-02")
+	ROptions["CurrentDay"] = time.Now().Format("2006-01-02T15:04:05")
+	ROptions["Date"] = time.Now().Format("2006-01-02T15:04:05")
+	ROptions["TimeStamp"] = utils.GetTS()
+	ROptions["TS"] = time.Now().Format("2006-01-02") + "T" + utils.GetTS()
+
+	/* --- start to load default Env --- */
+	// ~/osmedeus-base
+	ROptions["BaseFolder"] = utils.NormalizePath(strings.TrimLeft(options.Env.BaseFolder, "/"))
+	ROptions["Plugins"] = options.Env.BinariesFolder
+	ROptions["Binaries"] = options.Env.BinariesFolder
+
+	ROptions["Backup"] = options.Env.BackupFolder
+	ROptions["Data"] = options.Env.DataFolder
+	ROptions["Workflow"] = options.Env.WorkFlowsFolder
+	ROptions["Scripts"] = options.Env.WorkFlowsFolder
+	ROptions["Cloud"] = options.Env.CloudConfigFolder
+
+	ROptions["Workspaces"] = options.Env.WorkspacesFolder
+	if options.Scan.BaseWorkspace != "" {
+		ROptions["Workspaces"] = options.Scan.BaseWorkspace
+	}
+	ROptions["Storages"] = options.Env.StoragesFolder
+	/* --- end of load default Env --- */
+
+	ROptions["Workspace"] = utils.CleanPath(raw)
+	if options.Scan.CustomWorkspace != "" {
+		ROptions["Workspace"] = utils.CleanPath(options.Scan.CustomWorkspace)
+	}
+	ROptions["Output"] = path.Join(ROptions["Workspaces"], ROptions["Workspace"])
+
+	// params in workflow file
+	if len(options.Flow.Params) > 0 {
+		for _, param := range options.Flow.Params {
+			for k, v := range param {
+				v = ResolveData(v, ROptions)
+				if strings.HasPrefix(v, "~/") {
+					v = utils.NormalizePath(v)
+				}
+				ROptions[k] = v
+			}
+		}
+	}
+
+	return ROptions
+}
+
+// ParseParams parse more params from cli
+func ParseParams(rawParams []string) map[string]string {
+	params := make(map[string]string)
+	for _, item := range rawParams {
+		if strings.Contains(item, "=") {
+			data := strings.Split(item, "=")
+			params[data[0]] = strings.Replace(item, data[0]+"=", "", -1)
+		}
+	}
+	return params
+}
+
+// ParseTarget parsing target and some variable for template
+func ParseTarget(raw string) map[string]string {
+	target := make(map[string]string)
+	if raw == "" {
+		return target
+	}
+	target["Target"] = raw
+	u, err := url.Parse(raw)
+
+	// something wrong so parsing it again
+	if err != nil || u.Scheme == "" || strings.Contains(u.Scheme, ".") {
+		raw = fmt.Sprintf("https://%v", raw)
+		u, err = url.Parse(raw)
+		if err != nil {
+			return target
+		}
+		// fmt.Println("parse again")
+	}
+	var hostname string
+	var query string
+	port := u.Port()
+	// var domain string
+	domain := u.Hostname()
+
+	query = u.RawQuery
+	if u.Port() == "" {
+		if strings.Contains(u.Scheme, "https") {
+			port = "443"
+		} else {
+			port = "80"
+		}
+
+		hostname = u.Hostname()
+	} else {
+		// ignore common port in Host
+		if u.Port() == "443" || u.Port() == "80" {
+			hostname = u.Hostname()
+		} else {
+			hostname = u.Hostname() + ":" + u.Port()
+		}
+	}
+
+	target["Scheme"] = u.Scheme
+	target["Path"] = u.Path
+	target["Domain"] = domain
+
+	target["Org"] = domain
+	suffix, ok := publicsuffix.PublicSuffix(domain)
+	if ok {
+		target["Org"] = strings.Replace(domain, fmt.Sprintf(".%s", suffix), "", -1)
+	} else {
+		if strings.Contains(domain, ".") {
+			parts := strings.Split(domain, ".")
+			if len(parts) == 2 {
+				target["Org"] = parts[0]
+			} else {
+				target["Org"] = parts[len(parts)-2]
+			}
+		}
+	}
+
+	target["Host"] = hostname
+	target["Port"] = port
+	target["RawQuery"] = query
+
+	if (target["RawQuery"] != "") && (port == "80" || port == "443") {
+		target["URL"] = fmt.Sprintf("%v://%v%v?%v", target["Scheme"], target["Host"], target["Path"], target["RawQuery"])
+	} else if port != "80" && port != "443" {
+		target["URL"] = fmt.Sprintf("%v://%v:%v%v?%v", target["Scheme"], target["Domain"], target["Port"], target["Path"], target["RawQuery"])
+	} else {
+		target["URL"] = fmt.Sprintf("%v://%v%v", target["Scheme"], target["Host"], target["Path"])
+	}
+
+	uu, _ := url.Parse(raw)
+	target["BaseURL"] = fmt.Sprintf("%v://%v", uu.Scheme, uu.Host)
+	target["Extension"] = filepath.Ext(target["BaseURL"])
+
+	return target
+}
+
+func IsRootDomain(raw string) bool {
+	suffix, ok := publicsuffix.PublicSuffix(raw)
+	if ok {
+		return false
+	}
+
+	input := strings.ReplaceAll(raw, fmt.Sprintf(".%s", suffix), "")
+	if strings.Count(input, ".") == 1 && strings.Count(input, "/") == 0 {
+		return true
+	}
+	return false
+}

core/parse_test.go

@@ -0,0 +1,71 @@
+package core
+
+import (
+	"fmt"
+	"github.com/flosch/pongo2/v6"
+	"runtime"
+	"testing"
+)
+
+func TestParseTarget(t *testing.T) {
+	fmt.Println("----> INPUT:", "http://example.com")
+	result := ParseTarget("http://exmaple.com")
+	fmt.Println(result)
+	if len(result) == 0 {
+		t.Errorf("Error RunMasscan")
+	}
+	// case 2
+	fmt.Println("----> INPUT:", "example.com")
+	result = ParseTarget("exmaple.com")
+	fmt.Println(result)
+	if len(result) == 0 {
+		t.Errorf("Error RunMasscan")
+	}
+
+	// case 2
+	fmt.Println("----> INPUT:", "http://exmaple.com/123?q=1")
+	result = ParseTarget("http://exmaple.com/123?q=1")
+	fmt.Println(result)
+	if len(result) == 0 {
+		t.Errorf("Error RunMasscan")
+	}
+
+	// case 2
+	fmt.Println("----> INPUT:", "1.2.3.4")
+	result = ParseTarget("1.2.3.4")
+	fmt.Println(result)
+	if len(result) == 0 {
+		t.Errorf("Error RunMasscan")
+	}
+
+	// case 2
+	fmt.Println("----> INPUT:", "1.2.3.4/24")
+	result = ParseTarget("1.2.3.4/24")
+	fmt.Println(result)
+	if len(result) == 0 {
+		t.Errorf("Error RunMasscan")
+	}
+}
+
+func TestRenderTemplate(t *testing.T) {
+	formatString := "Hello {{name}}! \n"
+	formatString += "--> Express: {{ 2 * cpu }} \n"
+	formatString += "--> Bool: {{ Skip }} \n"
+	target := map[string]any{
+		"name": "World",
+		"num":  "2",
+		"cpu":  runtime.NumCPU(),
+		"Skip": true,
+	}
+	fmt.Println(target)
+
+	if tpl, err := pongo2.FromString(formatString); err == nil {
+		// Now you can render the template with the given
+		// pongo2.Context how often you want to.
+		out, err := tpl.Execute(target)
+		if err != nil {
+			panic(err)
+		}
+		fmt.Println(out) // Output: Hello Florian!
+	}
+}

core/queue.go

@@ -0,0 +1,147 @@
+package core
+
+import (
+	"fmt"
+	"github.com/fatih/color"
+	"github.com/fsnotify/fsnotify"
+	"github.com/j3ssie/osmedeus/libs"
+	"github.com/j3ssie/osmedeus/utils"
+	jsoniter "github.com/json-iterator/go"
+	"github.com/panjf2000/ants"
+	"github.com/thoas/go-funk"
+	"os"
+	"strings"
+	"sync"
+)
+
+func QueueWatcher(options libs.Options) {
+	queueFile := options.Queue.QueueFile
+
+	var wg sync.WaitGroup
+	p, _ := ants.NewPoolWithFunc(options.Concurrency, func(i interface{}) {
+		RunTheScan(i.(string), options)
+		wg.Done()
+	}, ants.WithPreAlloc(true))
+	defer p.Release()
+
+	data := utils.ReadingFileUnique(queueFile)
+
+	/* Start the watcher */
+
+	// Create new watcher.
+	watcher, err := fsnotify.NewWatcher()
+	if err != nil {
+		panic(err)
+	}
+	defer watcher.Close()
+	utils.InforF("Starting to watch the queue file: %v", color.HiMagentaString(queueFile))
+
+	// Start listening for events.
+	go func() {
+		for {
+			select {
+			case event, ok := <-watcher.Events:
+				if !ok {
+					return
+				}
+				if event.Op.String() == "WRITE" {
+					utils.DebugF(color.HiMagentaString("modified file: %v -- %v", event.Name, event.Op))
+					target := GetNewLine(queueFile)
+					wg.Add(1)
+					_ = p.Invoke(strings.TrimSpace(target))
+				}
+
+				if event.Op.String() == "REMOVE" {
+					utils.ErrorF("Queue file removed, exiting ...")
+					os.Exit(-1)
+				}
+			case err, ok := <-watcher.Errors:
+				if !ok {
+					return
+				}
+				utils.ErrorF("error: %v", err)
+			}
+		}
+	}()
+
+	err = watcher.Add(queueFile)
+	if err != nil {
+		panic(err)
+	}
+
+	// Just to trigger the file events
+	utils.WriteToFile(queueFile, strings.Join(append(data, "\n"), "\n"))
+
+	// Block main goroutine forever.
+	<-make(chan struct{})
+}
+
+func RunTheScan(target string, options libs.Options) error {
+	if strings.TrimSpace(target) == "" {
+		return fmt.Errorf("target is empty")
+	}
+	utils.InforF("Picking the target from the queue: %v", color.CyanString(target))
+
+	var inputFormat libs.InputFormat
+	if ok := jsoniter.UnmarshalFromString(target, &inputFormat); ok == nil {
+		utils.DebugF("Parsing the input in JSON format: %v", color.CyanString(target))
+		cmd := CommandBuilder(inputFormat)
+		utils.InforF("Running the command: %v", color.CyanString(cmd))
+		utils.RunOSCommand(cmd)
+		return nil
+	}
+
+	runner, err := InitRunner(target, options)
+	if err != nil {
+		utils.ErrorF("Error init runner with: %s", target)
+		return err
+	}
+	runner.Start()
+
+	return nil
+}
+
+func GetNewLine(queueFile string) string {
+	data := utils.ReadingLines(queueFile)
+	if len(data) == 0 {
+		return ""
+	}
+
+	target := data[0]
+	data = funk.DropString(data, 1)
+	utils.DebugF("Getting the target from the queue file: %v -- %v", queueFile, color.CyanString(target))
+	utils.WriteToFile(queueFile, strings.Join(data, "\n"))
+	return target
+}
+
+func CommandBuilder(inputFormat libs.InputFormat) (command string) {
+	if inputFormat.Command == "" {
+		inputFormat.Command = fmt.Sprintf("%v scan -t %v", libs.BINARY, inputFormat.Input)
+		if inputFormat.InputAsFile {
+			inputFormat.Command = fmt.Sprintf("%v scan -T %v", libs.BINARY, inputFormat.Input)
+		}
+
+		if inputFormat.Flow == "" {
+			inputFormat.Command += " -f " + inputFormat.Flow
+		}
+
+		// append the modules
+		if len(inputFormat.Modules) > 0 {
+			for _, item := range inputFormat.Modules {
+				inputFormat.Command += " -m " + item
+			}
+		}
+
+		if len(inputFormat.Params) > 0 {
+			for _, item := range inputFormat.Params {
+				inputFormat.Command += " -p " + item
+			}
+		}
+
+		inputFormat.Command += " " + inputFormat.Extra
+	}
+
+	// formatting the command if there is any input in it
+	inputFormat.Command = strings.ReplaceAll(inputFormat.Command, "{{.input}}", inputFormat.Input)
+	return command
+}

core/reference.go

@@ -0,0 +1,104 @@
+package core
+
+/* File to store all the script for better reference */
+
+const (
+	Cleaning         = "Cleaning"
+	CleanAmass       = "CleanAmass"
+	CleanRustScan    = "CleanRustScan"
+	CleanGoBuster    = "CleanGoBuster"
+	CleanMassdns     = "CleanMassdns"
+	CleanSWebanalyze = "CleanSWebanalyze"
+	CleanJSONDnsx    = "CleanJSONDnsx"
+	CleanWebanalyze  = "CleanWebanalyze"
+	CleanArjun       = "CleanArjun"
+	GenNucleiReport  = "GenNucleiReport"
+	CleanJSONHttpx   = "CleanJSONHttpx"
+	CleanFFUFJson    = "CleanFFUFJson"
+)
+
+const (
+	// noti for slack
+	StartNoti   = "StartNoti"
+	DoneNoti    = "DoneNoti"
+	ReportNoti  = "ReportNoti"
+	DiffNoti    = "DiffNoti"
+	CustomNoti  = "CustomNoti"
+	NotiFile    = "NotiFile"
+	WebHookNoti = "WebHookNoti"
+	// noti for telegram
+	TeleMess       = "TeleMess"
+	TeleMessWrap   = "TeleMessWrap"
+	TeleMessByFile = "TeleMessByFile"
+	TeleSendFile   = "TeleSendFile"
+)
+
+const (
+	ExecCmd           = "ExecCmd"
+	ExecCmdB          = "ExecCmdB"
+	ExecCmdWithOutput = "ExecCmdWithOutput"
+	ExecContain       = "ExecContain"
+	Sleep             = "Sleep"
+	Exit              = "Exit"
+	CastToInt         = "CastToInt"
+	StripSlash        = "StripSlash"
+	Printf            = "Printf"
+	Cat               = "Cat"
+	SortU             = "SortU"
+	SplitFile         = "SplitFile"
+	Append            = "Append"
+	Copy              = "Copy"
+	CreateFolder      = "CreateFolder"
+	DeleteFile        = "DeleteFile"
+	DeleteFolder      = "DeleteFolder"
+	SplitFileByPart   = "SplitFileByPart"
+	FileLength        = "FileLength"
+	IsFile            = "IsFile"
+	EmptyDir          = "EmptyDir"
+	EmptyFile         = "EmptyFile"
+	ReadLines         = "ReadLines"
+	Compress          = "Compress"
+	Decompress        = "Decompress"
+)
+
+const (
+	TotalSubdomain     = "TotalSubdomain"
+	TotalDns           = "TotalDns"
+	TotalScreenShot    = "TotalScreenShot"
+	TotalTech          = "TotalTech"
+	TotalVulnerability = "TotalVulnerability"
+	TotalArchive       = "TotalArchive"
+	TotalLink          = "TotalLink"
+	TotalDirb          = "TotalDirb"
+	CreateReport       = "CreateReport"
+)
+
+const (
+	RRSync      = "RRSync"
+	Clone       = "Clone"
+	FClone      = "FClone"
+	PushResult  = "PushResult"
+	PushFolder  = "PushFolder"
+	PullFolder  = "PullFolder"
+	DiffCompare = "DiffCompare"
+	GitDiff     = "GitDiff"
+	LoopGitDiff = "LoopGitDiff"
+	// for gitlab API only
+	CreateRepo      = "CreateRepo"
+	DeleteRepo      = "DeleteRepo"
+	DeleteRepoByPid = "DeleteRepoByPid"
+	ListProjects    = "ListProjects"
+)
+
+const (
+	UploadToS3        = "UploadToS3"
+	DownloadFromS3    = "DownloadFromS3"
+	DownloadFile      = "DownloadFile"
+	GenMarkdownReport = "GenMarkdownReport"
+)
+
+const (
+	SetVar   = "SetVar"
+	SetOSVar = "SetOSVar"
+	GetOSEnv = "GetOSEnv"
+)

core/report.go

@@ -0,0 +1,215 @@
+package core
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"github.com/Jeffail/gabs/v2"
+	"github.com/fatih/color"
+	"github.com/j3ssie/osmedeus/libs"
+	"github.com/j3ssie/osmedeus/utils"
+	"github.com/olekukonko/tablewriter"
+	"github.com/spf13/cast"
+)
+
+func ListWorkspaces(options libs.Options) (content [][]string) {
+	workspaces, err := os.ReadDir(utils.NormalizePath(options.Env.WorkspacesFolder))
+	if err != nil {
+		utils.ErrorF("Error reading workspaces folder: %s", err)
+		return content
+	}
+
+	for _, ws := range workspaces {
+		if ws.IsDir() {
+			status := "unknown"
+			flowName := "unknown"
+			progress := "N/A"
+			wsFolder := path.Join(utils.NormalizePath(options.Env.WorkspacesFolder), ws.Name())
+
+			if utils.DirLength(wsFolder) == 0 {
+				continue
+			}
+
+			if utils.FileExists(path.Join(wsFolder, "done")) {
+				status = "done"
+			}
+
+			runtimeFile := path.Join(wsFolder, "runtime")
+			if utils.FileExists(runtimeFile) {
+				utils.DebugF("Reading information from: %v", runtimeFile)
+				runtimeContent := utils.GetFileContent(runtimeFile)
+
+				if jsonParsed, ok := gabs.ParseJSON([]byte(runtimeContent)); ok == nil {
+					flowName = cast.ToString(jsonParsed.S("task_name").Data())
+					doneStep := cast.ToString(jsonParsed.S("done_step").Data())
+					totalSteps := cast.ToString(jsonParsed.S("total_steps").Data())
+					isRunning := cast.ToString(jsonParsed.S("is_running").Data())
+
+					if isRunning == "true" {
+						status = "running"
+					}
+
+					progress = color.HiCyanString(fmt.Sprintf(doneStep + "/" + totalSteps))
+				}
+			}
+
+			row := []string{
+				path.Base(color.HiMagentaString(ws.Name())), status, flowName, progress, wsFolder,
+			}
+			content = append(content, row)
+		}
+	}
+
+	table := tablewriter.NewWriter(os.Stderr)
+	table.SetAutoFormatHeaders(false)
+	table.SetHeader([]string{"Workspace Name", "Status", "Routine", "Progress", "Workspace Path"})
+	table.SetBorders(tablewriter.Border{Left: true, Top: true, Right: true, Bottom: true})
+	table.SetColWidth(120)
+	table.AppendBulk(content)
+	table.Render()
+
+	fmt.Println(color.HiGreenString("📁 Total Workspaces: ") + color.HiMagentaString("%v", len(content)))
+	usage := color.HiWhiteString("💡 How to view report:") + color.HiGreenString(" osmedeus report view -t %v", color.HiMagentaString("[targetName]"))
+	fmt.Println(usage)
+
+	return content
+}
+
+func ListSingleWorkspace(options libs.Options, target string) (content [][]string) {
+	workspaces, err := os.ReadDir(utils.NormalizePath(options.Env.WorkspacesFolder))
+	if err != nil {
+		utils.ErrorF("Error reading workspaces folder: %s", err)
+		return content
+	}
+
+	header := []string{"Module", "Report Name", "Report Path"}
+	for _, ws := range workspaces {
+		if !ws.IsDir() {
+			continue
+		}
+		// compare target name with workspace name
+		if target != path.Base(ws.Name()) {
+			continue
+		}
+		wsFolder := path.Join(utils.NormalizePath(options.Env.WorkspacesFolder), ws.Name())
+
+		runtimeFile := path.Join(wsFolder, "runtime")
+		// only listing file that in report part
+		if utils.FileExists(runtimeFile) && !options.Report.Raw {
+			utils.InforF("Reading runtime information from: %v", runtimeFile)
+			runtimeContent := utils.GetFileContent(runtimeFile)
+
+			isImported := false
+			if !strings.Contains(runtimeContent, options.Env.WorkspacesFolder) {
+				isImported = true
+			}
+
+			// replace the workspace folder if it doesn't exist
+			if !strings.Contains(runtimeContent, options.Env.WorkspacesFolder) {
+				homeFolder := "/root/.osmedeus/workspaces/"
+				if strings.Contains(runtimeContent, "/root/workspaces-osmedeus/") {
+					homeFolder = "/root/workspaces-osmedeus/"
+				}
+				runtimeContent = strings.ReplaceAll(runtimeContent, homeFolder, options.Env.WorkspacesFolder+"/")
+			}
+
+			if strings.Contains(runtimeContent, "/root/.osmedeus/workspaces") {
+				runtimeContent = strings.ReplaceAll(runtimeContent, "/root/.osmedeus/workspaces", options.Env.WorkspacesFolder)
+			}
+
+			row := []string{"==> Workspace Name", color.HiGreenString(ws.Name()), color.HiGreenString(wsFolder)}
+			content = append(content, row)
+
+			if jsonParsed, ok := gabs.ParseJSON([]byte(runtimeContent)); ok == nil {
+				reports := jsonParsed.S("target", "reports").Children()
+
+				for _, report := range reports {
+					moduleName := cast.ToString(report.S("module").Data())
+					reportName := cast.ToString(report.S("report_name").Data())
+					reportPath := cast.ToString(report.S("report_path").Data())
+
+					if !utils.FileExists(reportPath) {
+						// /root/.osmedeus/workspaces
+						continue
+					}
+
+					row := []string{
+						moduleName, processReport(options, reportName), processReport(options, reportPath),
+					}
+
+					content = append(content, row)
+				}
+
+				markDownSunmmary := cast.ToString(jsonParsed.S("markdown_summary").Data())
+				if utils.FileExists(markDownSunmmary) {
+					row := []string{"==> Markdown Summary", path.Base(markDownSunmmary), color.HiGreenString(markDownSunmmary)}
+					content = append(content, row)
+				}
+				markDownReport := cast.ToString(jsonParsed.S("markdown_report").Data())
+				if utils.FileExists(markDownReport) {
+					row := []string{"==> Markdown HTML Summary", path.Base(markDownReport), color.HiGreenString(markDownReport)}
+					content = append(content, row)
+				}
+
+				sep := []string{"==> --------", color.HiGreenString("----------"), color.HiGreenString("-----------")}
+				content = append(content, sep)
+
+				if len(content) <= 2 && isImported {
+					utils.WarnF("Workspace folder not found in runtime file, you might extract it from different machine that being run on different user")
+					utils.WarnF("💡 If you still have problem, please view it as raw format: %s", color.HiGreenString("osmedeus view -t %s --raw", target))
+				}
+
+			}
+			continue
+		}
+
+		// --raw flag: list all avaliable file
+		header = []string{"Report Name", "Report Path"}
+		if options.Report.Static {
+			header = []string{"Report Name", "Report URL"}
+		}
+		row := []string{"==> Workspace Name", color.HiGreenString(ws.Name())}
+		content = append(content, row)
+
+		filepath.Walk(wsFolder, func(reportPath string, _ os.FileInfo, err error) error {
+			reportName := path.Base(reportPath)
+			row := []string{processReport(options, reportName), processReport(options, reportPath)}
+			content = append(content, row)
+			return nil
+		})
+
+		sep := []string{"<== --------", color.HiGreenString("-----------")}
+		content = append(content, sep)
+
+	}
+
+	table := tablewriter.NewWriter(os.Stderr)
+	table.SetAutoFormatHeaders(true)
+	table.SetHeader(header)
+	table.SetBorders(tablewriter.Border{Left: true, Top: true, Right: true, Bottom: true})
+	table.SetColWidth(120)
+	table.AppendBulk(content)
+	table.SetHeaderLine(true)
+	table.Render()
+
+	return content
+}
+
+func processReport(options libs.Options, reportPath string) string {
+	if options.Report.Static {
+		base := fmt.Sprintf("https://%s:8000/%s/workspaces", options.Report.PublicIP, options.Server.StaticPrefix)
+		reportPath = strings.ReplaceAll(reportPath, options.Env.WorkspacesFolder, base)
+	}
+
+	if strings.HasSuffix(reportPath, ".html") {
+		reportPath = color.HiCyanString(reportPath)
+	}
+	if strings.HasSuffix(reportPath, ".json") {
+		reportPath = color.HiBlueString(reportPath)
+	}
+
+	return reportPath
+}

core/runner.go

@@ -0,0 +1,373 @@
+package core
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"sync"
+
+	"github.com/Shopify/yaml"
+	"github.com/fatih/color"
+	"github.com/j3ssie/osmedeus/database"
+	"github.com/j3ssie/osmedeus/execution"
+	"github.com/j3ssie/osmedeus/libs"
+	"github.com/j3ssie/osmedeus/utils"
+	"github.com/panjf2000/ants"
+	"github.com/robertkrimen/otto"
+	"github.com/thoas/go-funk"
+)
+
+// Runner runner struct to start a job
+type Runner struct {
+	Input     string
+	Workspace string
+
+	InputType     string // domain, url, ip, cidr or domain-file, url-file, ip-file, cidr-file
+	RequiredInput string // this should match with InputType
+	IsInvalid     bool
+	ForceParams   bool
+
+	RoutineType  string // module or flow
+	RoutineName  string // general
+	RoutinePath  string
+	RunnerSource string // cli or  api
+	RunnerType   string // local or cloud
+
+	Opt libs.Options
+
+	// use for analytics
+	DoneStep      int
+	TotalSteps    int
+	RunningTime   int
+	CurrentModule string
+
+	DoneFile        string
+	RuntimeFile     string
+	WorkspaceFolder string
+
+	RoutineModules []string
+	Reports        []string
+	Routines       []libs.Routine
+
+	VM        *otto.Otto
+	TargetObj database.Target
+	ScanObj   database.Scan
+
+	Target map[string]string
+	// this is same as targets but won't change during the execution time
+	Params map[string]string
+}
+
+// InitRunner init runner
+func InitRunner(input string, opt libs.Options) (Runner, error) {
+	var runner Runner
+	runner.Input = input
+	runner.Opt = opt
+	runner.PrepareRoutine()
+	runner.InitVM()
+
+	runner.RunnerSource = "cli"
+	// @TODO check if running in cloud
+	runner.RunnerType = "local"
+	return runner, nil
+}
+
+// PrepareWorkflow prepare workflow file
+func (r *Runner) PrepareWorkflow() {
+	allFlows := ListAllFlowName(r.Opt)
+
+	r.RoutineType = "flow"
+	var err error
+	flows := SelectFlow(r.Opt.Scan.Flow, r.Opt)
+	for _, flow := range flows {
+		if strings.TrimSpace(flow) == "" {
+			continue
+		}
+		r.Opt.Flow, err = ParseFlow(flow)
+		r.RoutinePath = flow
+		if err != nil {
+			continue
+		}
+
+		if r.Opt.Flow.NoDB {
+			r.Opt.NoDB = true
+		}
+
+		if r.Opt.Flow.Type == "general" {
+			r.Opt.Flow.Input = r.Opt.Scan.Input
+		}
+		// default folder to look for module file if not specify
+		r.Opt.Flow.DefaultType = r.Opt.Flow.Type
+
+		r.Target["FlowPath"] = flow
+		r.RequiredInput = r.Opt.Flow.Validator
+		r.ForceParams = r.Opt.Flow.ForceParams
+
+		// get more params from flow
+		if len(r.Opt.Flow.Params) > 0 {
+			for _, params := range r.Opt.Flow.Params {
+				for k, v := range params {
+					r.Target[k] = v
+				}
+			}
+		}
+	}
+
+	// generate routines
+	for _, routine := range r.Opt.Flow.Routines {
+		// select module depend on the flow type
+		if routine.FlowFolder != "" {
+			r.Opt.Flow.Type = routine.FlowFolder
+		} else {
+			r.Opt.Flow.Type = r.Opt.Flow.DefaultType
+		}
+
+		modules := SelectModules(routine.Modules, r.Opt)
+		routine.RoutineName = fmt.Sprintf("flow-%s", r.Opt.Flow.Name)
+
+		for _, module := range modules {
+			parsedModule, err := ParseModules(module)
+			if err != nil || parsedModule.Name == "" {
+				continue
+			}
+			r.TotalSteps += len(parsedModule.Steps)
+			routine.ParsedModules = append(routine.ParsedModules, parsedModule)
+		}
+		r.Routines = append(r.Routines, routine)
+	}
+
+	if len(r.Routines) == 0 {
+		if r.Opt.Scan.Flow != "cloud-distributed" {
+			utils.WarnF("Your workflow %v doesn't exist", color.HiRedString(r.Opt.Scan.Flow))
+			utils.WarnF("Please select one of these flow: %v", color.HiMagentaString(strings.Join(allFlows, ", ")))
+		}
+	}
+}
+
+func (r *Runner) PrepareModule() {
+	allModules := ListModuleName(r.Opt)
+	r.RoutineType = "module"
+
+	var err error
+	for _, rawModule := range r.Opt.Scan.Modules {
+		var routine libs.Routine
+
+		module := DirectSelectModule(r.Opt, rawModule)
+		r.RoutinePath = rawModule
+		r.Opt.Module, err = ParseModules(module)
+		if err != nil || r.Opt.Module.Name == "" {
+			utils.WarnF("Your module %v doesn't exist", color.HiRedString(r.Opt.Scan.Modules[0]))
+			utils.WarnF("Please select one of these module: %v", color.HiMagentaString(strings.Join(allModules, ", ")))
+			continue
+		}
+		if r.Opt.Module.NoDB {
+			r.Opt.NoDB = true
+		}
+
+		r.Target["FlowPath"] = "direct-module"
+		routine.ParsedModules = append(routine.ParsedModules, r.Opt.Module)
+		routine.RoutineName = fmt.Sprintf("module-%s", r.Opt.Flow.Name)
+		r.Target["Module"] = module
+
+		r.Routines = append(r.Routines, routine)
+
+		r.TotalSteps += len(r.Opt.Module.Steps)
+		r.RequiredInput = r.Opt.Module.Validator
+	}
+}
+
+func (r *Runner) PrepareRoutine() {
+	// prepare targets
+	r.Target = ParseInput(r.Input, r.Opt)
+	r.Workspace = r.Target["Workspace"]
+
+	// take from -m flag
+	if len(r.Opt.Scan.Modules) > 0 {
+		r.RoutineName = strings.Join(r.Opt.Scan.Modules, "-")
+		r.PrepareModule()
+		return
+	}
+
+	// take from -f flag
+	if r.Opt.Scan.Flow != "" {
+		r.RoutineName = r.Opt.Scan.Flow
+		r.PrepareWorkflow()
+	}
+}
+
+// PrepareParams prepare global params
+func (r *Runner) PrepareParams() {
+	r.Params = r.Target
+
+	// looking for more params from each module
+	for _, routine := range r.Routines {
+		for _, module := range routine.ParsedModules {
+			// params from module file
+			if len(module.Params) > 0 {
+				for _, param := range module.Params {
+					for k, v := range param {
+						// skip params if override: false
+						_, exist := r.Params[k]
+						if r.ForceParams && exist {
+							utils.DebugF("Skip override param: %v --> %v", k, v)
+							continue
+						}
+
+						v = ResolveData(v, r.Params)
+						if strings.HasPrefix(v, "~/") {
+							v = utils.NormalizePath(v)
+						}
+						r.Params[k] = v
+					}
+				}
+			}
+
+			if len(r.Opt.Scan.ParamsFile) > 0 {
+				var params map[string]string
+				yamlFile, err := os.ReadFile(r.Opt.Scan.ParamsFile)
+				if err != nil {
+					utils.ErrorF("YAML parsing err: %v -- #%v ", r.Opt.Scan.ParamsFile, err)
+					return
+				}
+				err = yaml.Unmarshal(yamlFile, &params)
+				if err != nil {
+					utils.ErrorF("Error unmarshal: %v -- %v", params, err)
+					return
+				}
+				if len(params) > 0 {
+					for k, v := range params {
+						v = ResolveData(v, r.Params)
+						r.Params[k] = v
+					}
+				}
+			}
+
+			// more params from -p flag which will override everything
+			if len(r.Opt.Scan.Params) > 0 {
+				params := ParseParams(r.Opt.Scan.Params)
+				if len(params) > 0 {
+					for k, v := range params {
+						v = ResolveData(v, r.Params)
+						r.Params[k] = v
+					}
+				}
+			}
+		}
+	}
+
+	r.ResolveRoutine()
+
+}
+
+// ResolveRoutine resolve the module name first
+func (r *Runner) ResolveRoutine() {
+	var routines []libs.Routine
+
+	for _, rawRoutine := range r.Routines {
+
+		var routine libs.Routine
+		for _, module := range rawRoutine.ParsedModules {
+			module = ResolveReports(module, r.Params)
+
+			r.Reports = append(r.Reports, module.Report.Final...)
+			module.PreRun = ResolveSlice(module.PreRun, r.Params)
+
+			// steps
+			for i, step := range module.Steps {
+				module.Steps[i].Timeout = ResolveData(step.Timeout, r.Params)
+				module.Steps[i].Threads = ResolveData(step.Threads, r.Params)
+				module.Steps[i].Label = ResolveData(step.Label, r.Params)
+				module.Steps[i].Std = ResolveData(step.Std, r.Params)
+				module.Steps[i].Source = ResolveData(step.Source, r.Params)
+
+				module.Steps[i].Conditions = ResolveSlice(step.Conditions, r.Params)
+				module.Steps[i].Required = ResolveSlice(step.Required, r.Params)
+
+				module.Steps[i].Commands = ResolveSlice(step.Commands, r.Params)
+				module.Steps[i].Scripts = ResolveSlice(step.Scripts, r.Params)
+
+				module.Steps[i].RCommands = ResolveSlice(step.RCommands, r.Params)
+				module.Steps[i].RScripts = ResolveSlice(step.RScripts, r.Params)
+				module.Steps[i].PConditions = ResolveSlice(step.PConditions, r.Params)
+				module.Steps[i].PScripts = ResolveSlice(step.PScripts, r.Params)
+				module.Steps[i].Ose = ResolveSlice(step.Ose, r.Params)
+			}
+
+			module.PostRun = ResolveSlice(module.PostRun, r.Params)
+			routine.ParsedModules = append(routine.ParsedModules, module)
+		}
+
+		routines = append(routines, routine)
+	}
+	r.Routines = routines
+}
+
+func (r *Runner) Start() {
+	err := r.Validator()
+	if err != nil {
+		utils.ErrorF("Input does not match the require type: %v -- %v", r.RequiredInput, r.Input)
+		utils.InforF("Adding %v flag if you want to disable input validate", color.HiCyanString(`'--nv'`))
+		return
+	}
+	utils.InforF("Running %s tactic with baseline threads hold as %s", color.YellowString(r.Opt.Tactics), color.HiMagentaString("%v", r.Opt.Threads))
+
+	r.Opt.Scan.ROptions = r.Target
+	// prepare some metadata files
+	utils.MakeDir(r.Target["Output"])
+	r.DoneFile = r.Target["Output"] + "/done"
+	r.RuntimeFile = r.Target["Output"] + "/runtime"
+	r.WorkspaceFolder = r.Target["Output"]
+	os.Remove(r.DoneFile)
+
+	utils.InforF("Running the routine %v on %v", color.HiYellowString(r.RoutineName), color.CyanString(r.Input))
+	utils.InforF("Detailed runtime file can be found on %v", color.CyanString(r.RuntimeFile))
+	execution.TeleSendMess(r.Opt, fmt.Sprintf("%s -- Start new scan: %s -- %s", r.Opt.Noti.ClientName, r.Opt.Scan.Flow, r.Target["Workspace"]), "#status", false)
+
+	r.DBNewTarget()
+	r.DBNewScan()
+	r.LoadEngineScripts()
+
+	r.PrepareParams()
+
+	/////
+	/* really start the scan here */
+	r.StartRoutines()
+	/////
+
+	r.DBDoneScan()
+	utils.BlockF("Finished", fmt.Sprintf("The scan for %v was completed within %v", color.HiCyanString(r.Input), color.HiMagentaString("%vs", r.RunningTime)))
+
+	if r.Opt.EnableBackup {
+		r.BackupWorkspace()
+	}
+}
+
+// StartRoutines start the scan
+func (r *Runner) StartRoutines() {
+	for _, routine := range r.Routines {
+		// start each section of modules
+		r.RunRoutine(routine.ParsedModules)
+	}
+}
+
+func (r *Runner) RunRoutine(modules []libs.Module) {
+	var wg sync.WaitGroup
+	p, _ := ants.NewPoolWithFunc(r.Opt.Concurrency*10, func(m interface{}) {
+		module := m.(libs.Module)
+		r.RunModule(module)
+		wg.Done()
+	}, ants.WithPreAlloc(true))
+	defer p.Release()
+
+	for _, module := range modules {
+		if funk.ContainsString(r.Opt.Exclude, module.Name) {
+			utils.BadBlockF("Module-Excluded", fmt.Sprintf("Module %v has been excluded", color.CyanString(module.Name)))
+			continue
+		}
+
+		p.Invoke(module)
+		wg.Add(1)
+	}
+
+	wg.Wait()
+}

core/runtime.go

@@ -0,0 +1,488 @@
+package core
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/fatih/color"
+	"github.com/j3ssie/osmedeus/execution"
+	"github.com/j3ssie/osmedeus/utils"
+	"github.com/robertkrimen/otto"
+	"github.com/spf13/cast"
+)
+
+// InitVM init scripting engine
+func (r *Runner) InitVM() {
+	r.VM = otto.New()
+	r.LoadEngineScripts()
+}
+
+func (r *Runner) ExecScript(script string) string {
+	utils.DebugF("[Run-Scripts] %v", script)
+	value, err := r.VM.Run(script)
+	if err == nil {
+		out, nerr := value.ToString()
+		if nerr == nil {
+			return out
+		}
+	}
+
+	return ""
+}
+
+// RunOse really start the runner
+func (r *Runner) RunOse(scriptName string) {
+	scriptContent := scriptName
+	if !strings.Contains(scriptName, "\n") {
+		scriptFile := SelectScript(scriptName, r.Opt)
+		if utils.FileExists(scriptFile) {
+			scriptContent = utils.GetFileContent(scriptFile)
+		}
+	}
+
+	if len(scriptContent) == 0 {
+		utils.ErrorF("Error running script: %s", scriptName)
+		return
+	}
+
+	utils.DebugF("-- Start ose:\n\n%s", scriptName)
+	r.ExecScript(scriptContent)
+	utils.DebugF("-- Done ose: %s", scriptName)
+}
+
+func (r *Runner) ConditionExecScript(script string) bool {
+	utils.DebugF("[Run-Scripts] %v", script)
+	value, err := r.VM.Run(script)
+
+	if err == nil {
+		out, nerr := value.ToBoolean()
+		if nerr == nil {
+			return out
+		}
+	}
+
+	return false
+}
+
+func (r *Runner) LoadEngineScripts() {
+	r.LoadScripts()
+	r.LoadDBScripts()
+	// r.LoadImportScripts()
+	r.LoadExternalScripts()
+	r.LoadGitScripts()
+	r.LoadNotiScripts()
+}
+
+func (r *Runner) LoadScripts() string {
+	var output string
+	vm := r.VM
+
+	// set attribute
+	vm.Set("Target", r.Target)
+
+	// SetVar('length', 6)
+	vm.Set(SetVar, func(call otto.FunctionCall) otto.Value {
+		args := call.ArgumentList
+		varName := args[0].String()
+		value := args[1].String()
+		r.Target[varName] = value
+		return otto.Value{}
+	})
+
+	// Exit script used to exit the module
+	vm.Set(Exit, func(call otto.FunctionCall) otto.Value {
+		args := call.ArgumentList
+		output = fmt.Sprintf("exit(%v)", args[0].String())
+		utils.InforF("Exit Detected")
+		result, err := vm.ToValue(output)
+		if err == nil {
+			return result
+		}
+		return otto.Value{}
+	})
+
+	// ExecCmd execute command
+	vm.Set(ExecCmd, func(call otto.FunctionCall) otto.Value {
+		cmd := call.Argument(0).String()
+		_, err := utils.RunCommandWithErr(cmd)
+		var validate bool
+		if err != nil {
+			validate = true
+		}
+		result, err := vm.ToValue(validate)
+		if err != nil {
+			return otto.Value{}
+		}
+		return result
+	})
+
+	// Cat the file to stdout
+	vm.Set(Cat, func(call otto.FunctionCall) otto.Value {
+		filename := call.Argument(0).String()
+		utils.InforF("Showing the content of: %v", color.HiCyanString(filename))
+		utils.Cat(filename)
+		result, err := vm.ToValue(true)
+		if err != nil {
+			return otto.Value{}
+		}
+		return result
+	})
+
+	// ExecCmdB execute in the background
+	vm.Set(ExecCmdB, func(call otto.FunctionCall) otto.Value {
+		cmd := call.Argument(0).String()
+		go func() {
+			utils.RunOSCommand(cmd)
+		}()
+		result, _ := vm.ToValue(true)
+		return result
+	})
+
+	// ExecCmd execute command
+	vm.Set(ExecCmdWithOutput, func(call otto.FunctionCall) otto.Value {
+		utils.RunCommandSteamOutput(call.Argument(0).String())
+		result, err := vm.ToValue(true)
+		if err != nil {
+			return otto.Value{}
+		}
+		return result
+	})
+
+	// ExecCmd execute command
+	vm.Set(ExecContain, func(call otto.FunctionCall) otto.Value {
+		out := utils.RunCmdWithOutput(call.Argument(0).String())
+		expected := call.Argument(2).String()
+		validate := strings.Contains(out, expected)
+		result, err := vm.ToValue(validate)
+		if err != nil {
+			return otto.Value{}
+		}
+		return result
+	})
+
+	// CastToInt convert string to int
+	vm.Set(CastToInt, func(call otto.FunctionCall) otto.Value {
+		toInt := cast.ToInt(call.Argument(0).String())
+		result, err := vm.ToValue(toInt)
+		if err == nil {
+			return result
+		}
+		return otto.Value{}
+	})
+
+	vm.Set(FileLength, func(call otto.FunctionCall) otto.Value {
+		data := utils.FileLength(call.Argument(0).String())
+		utils.DebugF("FileLength -- %v", data)
+		result, err := vm.ToValue(data)
+		if err == nil {
+			return result
+		}
+		return otto.Value{}
+	})
+
+	vm.Set(IsFile, func(call otto.FunctionCall) otto.Value {
+		data := utils.FileLength(call.Argument(0).String())
+		var validate bool
+		if data > 1 {
+			validate = true
+		}
+		result, _ := vm.ToValue(validate)
+		return result
+	})
+
+	// StripSlash strip last '/' of URL
+	vm.Set(StripSlash, func(call otto.FunctionCall) otto.Value {
+		raw := call.Argument(0).String()
+		out := strings.Trim(raw, "/")
+		result, err := vm.ToValue(out)
+		if err != nil {
+			return otto.Value{}
+		}
+		return result
+	})
+
+	vm.Set(ReadLines, func(call otto.FunctionCall) otto.Value {
+		fileName := call.Argument(0).String()
+		data := utils.ReadingLines(fileName)
+		if len(data) > 0 {
+			result, err := vm.ToValue(data)
+			if err == nil {
+				return result
+			}
+		}
+		return otto.Value{}
+	})
+
+	// Printf simply print a string to console
+	vm.Set(Printf, func(call otto.FunctionCall) otto.Value {
+		fmt.Printf("%v\n", color.HiCyanString(call.Argument(0).String()))
+		returnValue, _ := otto.ToValue(true)
+		return returnValue
+	})
+
+	// split file to multiple
+	vm.Set(SplitFile, func(call otto.FunctionCall) otto.Value {
+		execution.SplitFile("size", call.ArgumentList)
+		return otto.Value{}
+	})
+
+	// split file to multiple
+	vm.Set(SplitFileByPart, func(call otto.FunctionCall) otto.Value {
+		execution.SplitFile("part", call.ArgumentList)
+		return otto.Value{}
+	})
+
+	vm.Set(Sleep, func(call otto.FunctionCall) otto.Value {
+		execution.Sleep(call.Argument(0).String())
+		returnValue, _ := otto.ToValue(true)
+		return returnValue
+	})
+
+	vm.Set(SortU, func(call otto.FunctionCall) otto.Value {
+		execution.SortU(call.Argument(0).String())
+		returnValue, _ := otto.ToValue(true)
+		return returnValue
+	})
+
+	vm.Set(Append, func(call otto.FunctionCall) otto.Value {
+		dest := call.Argument(0).String()
+		src := call.Argument(1).String()
+		execution.Append(dest, src)
+		returnValue, _ := otto.ToValue(true)
+		return returnValue
+	})
+
+	vm.Set(Decompress, func(call otto.FunctionCall) otto.Value {
+		dest := call.Argument(0).String()
+		src := call.Argument(1).String()
+		execution.Decompress(dest, src)
+		returnValue, _ := otto.ToValue(true)
+		return returnValue
+	})
+
+	vm.Set(Compress, func(call otto.FunctionCall) otto.Value {
+		dest := call.Argument(0).String()
+		src := call.Argument(1).String()
+		execution.Compress(dest, src)
+		returnValue, _ := otto.ToValue(true)
+		return returnValue
+	})
+
+	vm.Set(CreateFolder, func(call otto.FunctionCall) otto.Value {
+		utils.MakeDir(call.Argument(0).String())
+		return otto.Value{}
+	})
+
+	vm.Set(DeleteFile, func(call otto.FunctionCall) otto.Value {
+		execution.DeleteFile(call.Argument(0).String())
+		return otto.Value{}
+	})
+
+	vm.Set(DeleteFolder, func(call otto.FunctionCall) otto.Value {
+		execution.DeleteFolder(call.Argument(0).String())
+		return otto.Value{}
+	})
+
+	vm.Set(Copy, func(call otto.FunctionCall) otto.Value {
+		src := call.Argument(0).String()
+		dest := call.Argument(1).String()
+		execution.Copy(src, dest)
+		return otto.Value{}
+	})
+
+	vm.Set(GetOSEnv, func(call otto.FunctionCall) otto.Value {
+		args := call.ArgumentList
+		env := args[0].String()
+		defaultValue := env
+		if len(args) > 1 {
+			defaultValue = args[1].String()
+		}
+		utils.GetOSEnv(env, defaultValue)
+		return otto.Value{}
+	})
+
+	vm.Set(SetOSVar, func(call otto.FunctionCall) otto.Value {
+		args := call.ArgumentList
+		varName := strings.ToUpper(args[0].String())
+		defaultValue := varName
+		if len(args) > 1 {
+			defaultValue = args[1].String()
+		}
+		err := os.Setenv(varName, defaultValue)
+		if err != nil {
+			utils.ErrorF("Error setting environment variable: %v", err)
+		}
+		return otto.Value{}
+	})
+
+	vm.Set(EmptyDir, func(call otto.FunctionCall) otto.Value {
+		result, _ := vm.ToValue(utils.EmptyDir(call.Argument(0).String()))
+		return result
+	})
+
+	vm.Set(EmptyFile, func(call otto.FunctionCall) otto.Value {
+		result, err := vm.ToValue(utils.EmptyFile(call.Argument(0).String(), 0))
+		if err != nil {
+			return otto.Value{}
+		}
+		if len(call.ArgumentList) > 1 {
+			num, _ := call.Argument(0).ToInteger()
+			result, err = vm.ToValue(utils.EmptyFile(call.Argument(0).String(), int(num)))
+			if err != nil {
+				return otto.Value{}
+			}
+		}
+		return result
+	})
+
+	vm.Set(RRSync, func(call otto.FunctionCall) otto.Value {
+		vpsIP := call.Argument(0).String() // root@ipaddress
+		src := call.Argument(1).String()   // local path
+		dest := call.Argument(2).String()  // remote path
+		cmd := fmt.Sprintf("rsync -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s' -avzr --progress %s %s:%s", r.Opt.Cloud.SecretKey, src, vpsIP, dest)
+		r.RetryCommandWithExpectString(cmd, `bytes/sec`)
+		return otto.Value{}
+	})
+
+	r.VM = vm
+
+	return output
+}
+
+func (r *Runner) RetryCommandWithExpectString(cmd string, expectString string, timeoutRaw ...string) string {
+	timeout := "300s"
+	if len(timeoutRaw) > 0 {
+		timeout = timeoutRaw[0]
+	}
+	var out string
+
+	utils.DebugF("Retry command: %s", cmd)
+	for i := 0; i < r.Opt.Cloud.Retry; i++ {
+		if timeout == "000" {
+			out, _ = utils.RunOSCommand(cmd)
+		} else {
+			out = utils.RunCmdWithOutput(cmd, timeout)
+		}
+
+		if !strings.Contains(out, expectString) {
+			utils.DebugF(out)
+			time.Sleep(time.Duration(60*(i+1)) * time.Second)
+			continue
+		}
+		return out
+	}
+	return out
+}
+
+func (r *Runner) LoadNotiScripts() string {
+	var output string
+	vm := r.VM
+	options := r.Opt
+
+	// script for notification
+	vm.Set(StartNoti, func(call otto.FunctionCall) otto.Value {
+		execution.StatusNoti("start", options)
+		return otto.Value{}
+	})
+	vm.Set(DoneNoti, func(call otto.FunctionCall) otto.Value {
+		execution.StatusNoti("done", options)
+		return otto.Value{}
+	})
+	vm.Set(ReportNoti, func(call otto.FunctionCall) otto.Value {
+		execution.ReportNoti(call.ArgumentList, options)
+		return otto.Value{}
+	})
+	vm.Set(DiffNoti, func(call otto.FunctionCall) otto.Value {
+		execution.DiffNoti(call.ArgumentList, options)
+		return otto.Value{}
+	})
+	// CustomNoti("message here")
+	vm.Set(CustomNoti, func(call otto.FunctionCall) otto.Value {
+		execution.SendAttachment("custom", call.Argument(0).String(), options)
+		return otto.Value{}
+	})
+	// NotiFile("src")
+	vm.Set(NotiFile, func(call otto.FunctionCall) otto.Value {
+		execution.SendFile(call.Argument(0).String(), options.Noti.SlackReportChannel, options)
+		return otto.Value{}
+	})
+	// using webhook
+	vm.Set(WebHookNoti, func(call otto.FunctionCall) otto.Value {
+		args := call.ArgumentList
+		messContent := args[0].String()
+		messType := "custom"
+		if len(args) > 1 {
+			messType = args[0].String()
+			messContent = args[1].String()
+		}
+
+		execution.WebHookSendAttachment(options, messType, messContent)
+		return otto.Value{}
+	})
+
+	// Telegram functions
+
+	vm.Set(TeleMess, func(call otto.FunctionCall) otto.Value {
+		args := call.ArgumentList
+		messContent := args[0].String()
+		channelType := "general"
+		if len(args) > 1 {
+			channelType = args[0].String()
+			messContent = args[1].String()
+		}
+		execution.TeleSendMess(options, messContent, channelType, false)
+		return otto.Value{}
+	})
+	// send message but with inside ```
+	vm.Set(TeleMessWrap, func(call otto.FunctionCall) otto.Value {
+		args := call.ArgumentList
+		messContent := args[0].String()
+		channelType := "general"
+		if len(args) > 1 {
+			channelType = args[0].String()
+			messContent = args[1].String()
+		}
+		execution.TeleSendMess(options, messContent, channelType, true)
+		return otto.Value{}
+	})
+
+	vm.Set(TeleMessByFile, func(call otto.FunctionCall) otto.Value {
+		args := call.ArgumentList
+		fileName := args[0].String()
+		channelType := "general"
+		if len(args) > 1 {
+			channelType = args[0].String()
+			fileName = args[1].String()
+		}
+		if !utils.FileExists(fileName) {
+			utils.DebugF("File %s not found", fileName)
+			return otto.Value{}
+		}
+
+		messContent := utils.GetFileContent(fileName)
+		if len(messContent) > 4000 {
+			execution.TeleSendFile(options, fileName, channelType)
+		} else {
+			execution.TeleSendMess(options, messContent, channelType, true)
+		}
+
+		return otto.Value{}
+	})
+
+	vm.Set(TeleSendFile, func(call otto.FunctionCall) otto.Value {
+		args := call.ArgumentList
+		messContent := args[0].String()
+		channelType := "general"
+		if len(args) > 1 {
+			channelType = args[0].String()
+			messContent = args[1].String()
+		}
+		execution.TeleSendFile(options, messContent, channelType)
+		return otto.Value{}
+	})
+
+	return output
+
+}

core/step.go

@@ -0,0 +1,167 @@
+package core
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/fatih/color"
+	"github.com/j3ssie/osmedeus/libs"
+	"github.com/j3ssie/osmedeus/utils"
+	"github.com/thoas/go-funk"
+)
+
+func (r *Runner) RunModulesWithTimeout(timeoutRaw string, module libs.Module, options libs.Options) {
+	timeout := utils.CalcTimeout(timeoutRaw)
+	utils.InforF("Run module %v with %v seconds timeout", color.HiCyanString(module.Name), timeout)
+
+	c := context.Background()
+	deadline := time.Now().Add(time.Duration(timeout) * time.Second)
+	c, cancel := context.WithDeadline(c, deadline)
+	defer cancel()
+
+	go func() {
+		r.RunModule(module)
+		cancel()
+	}()
+
+	select {
+	case <-c.Done():
+		utils.DebugF("Module done")
+		return
+	case <-time.After(time.Duration(timeout) * time.Second):
+		utils.BadBlockF("timeout", fmt.Sprintf("Module got timeout after %v", color.HiMagentaString(timeoutRaw)))
+	}
+	return
+}
+
+// CheckResume check resume report
+func CheckResume(module libs.Module) bool {
+	for _, report := range module.Report.Final {
+		if !strings.Contains(report, ".osmedeus/storages") && !utils.FileExists(report) {
+			return false
+		}
+	}
+	return true
+}
+
+// ResolveReports resolve real path of reports
+func ResolveReports(module libs.Module, params map[string]string) libs.Module {
+	var final []string
+	var noti []string
+	var diff []string
+	for _, report := range module.Report.Final {
+		final = append(final, ResolveData(report, params))
+	}
+
+	for _, report := range module.Report.Noti {
+		noti = append(noti, ResolveData(report, params))
+	}
+
+	for _, report := range module.Report.Diff {
+		diff = append(diff, ResolveData(report, params))
+	}
+
+	module.Report.Final = final
+	module.Report.Noti = noti
+	module.Report.Diff = diff
+	return module
+}
+
+//  print all report
+func printReports(module libs.Module) {
+	var files []string
+	files = append(files, module.Report.Final...)
+	files = append(files, module.Report.Noti...)
+	files = append(files, module.Report.Diff...)
+
+	reports := funk.UniqString(files)
+	utils.BlockF("Report", color.HiCyanString("List of reports generated by the %v module", color.HiGreenString(module.Name)))
+	for _, report := range reports {
+		if !utils.FileExists(report) && utils.EmptyFile(report, 0) {
+			if !utils.FolderExists(report) && utils.EmptyDir(report) {
+				continue
+			}
+		}
+		utils.BlockF("report-file", report)
+	}
+}
+
+// CheckRequired check if required file exist or not
+func (r *Runner) CheckRequired(requires []string) error {
+	if len(requires) == 0 {
+		return nil
+	}
+
+	utils.DebugF("Checking requirements: %v", requires)
+	for _, require := range requires {
+
+		if strings.Contains(require, "(") && strings.Contains(require, ")") {
+			validate := r.ConditionExecScript(require)
+			if !validate {
+				utils.DebugF("Requirement not met: %v", require)
+				return fmt.Errorf("condition not met: %s", require)
+			}
+			continue
+		}
+
+		require = utils.NormalizePath(require)
+		if !utils.FileExists(require) && utils.EmptyFile(require, 0) {
+			if !utils.FolderExists(require) && utils.DirLength(require) == 0 {
+				utils.DebugF("Missing %v", require)
+				return fmt.Errorf("missing requirement")
+			}
+		}
+	}
+	return nil
+}
+
+// CheckCondition check if required file exist or not
+func (r *Runner) CheckCondition(conditions []string) error {
+	if len(conditions) == 0 {
+		return nil
+	}
+	for _, require := range conditions {
+		if !r.ConditionExecScript(require) {
+			return fmt.Errorf("condition not met: %s", require)
+		}
+	}
+	return nil
+}
+
+// RunCommands run list of commands in parallel
+func (r *Runner) RunCommands(commands []string, std string) string {
+	var wg sync.WaitGroup
+	var output string
+	var err error
+
+	for _, command := range commands {
+		wg.Add(1)
+		// don't run too much command at once
+		go func(command string) {
+			defer wg.Done()
+			var out string
+			if std != "" {
+				out, err = utils.RunOSCommand(command)
+			} else {
+				err = utils.RunCommandWithoutOutput(command)
+			}
+
+			if err != nil {
+				utils.DebugF("error running command: %v -- %v", command, err)
+			}
+
+			if out != "" {
+				output += out
+			}
+		}(command)
+	}
+	wg.Wait()
+
+	if std != "" {
+		utils.WriteToFile(std, output)
+	}
+	return output
+}

core/tmux.go

@@ -0,0 +1,85 @@
+package core
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/j3ssie/osmedeus/libs"
+	"github.com/j3ssie/osmedeus/utils"
+)
+
+type Tmux struct {
+	ApplyAll       bool
+	SelectedWindow string
+	Exclude        string
+	Limit          int
+	Windows        []string
+}
+
+func InitTmux(options libs.Options) (Tmux, error) {
+	cmd := "tmux ls"
+	var tmux Tmux
+	tmux.ApplyAll = options.Tmux.ApplyAll
+	tmux.SelectedWindow = options.Tmux.SelectedWindow
+	tmux.Exclude = options.Tmux.Exclude
+	tmux.Limit = options.Tmux.Limit
+
+	raw := utils.RunCmdWithOutput(cmd)
+	if strings.Contains(raw, "command not found") || !strings.Contains(raw, "\n") {
+		return tmux, fmt.Errorf("tmux program not installed")
+	}
+
+	stds := strings.Split(raw, "\n")
+	for _, line := range stds {
+		if strings.TrimSpace(line) == "" || !strings.Contains(line, " ") {
+			continue
+		}
+
+		data := strings.Split(line, " ")
+		tmux.Windows = append(tmux.Windows, strings.TrimRight(data[0], ":"))
+	}
+	return tmux, nil
+}
+
+func (t *Tmux) ListTmux() {
+	if len(t.Windows) == 0 {
+		fmt.Println("No tmux available")
+		return
+	}
+	fmt.Println(strings.Join(t.Windows, ", "))
+}
+
+func (t *Tmux) CatchSession() string {
+	var result string
+	if len(t.Windows) == 0 {
+		fmt.Println("No tmux available")
+		return result
+	}
+	for _, window := range t.Windows {
+		utils.DebugF("Get info of: %s", window)
+		if !t.ApplyAll {
+			if window != t.SelectedWindow {
+				continue
+			}
+		}
+		if strings.HasPrefix(window, t.Exclude) {
+			continue
+		}
+		utils.InforF("Get output of %s session", window)
+		cmd := fmt.Sprintf(`tmux capture-pane -pt "%s"`, window)
+		raw := utils.RunCmdWithOutput(cmd)
+
+		data := strings.Split(raw, "\n")
+		if t.Limit == 0 {
+			fmt.Println(raw)
+		} else if t.Limit < len(data) && t.Limit > 0 {
+			fmt.Println(strings.Join(data[len(data)-t.Limit:len(data)-1], "\n"))
+		} else {
+			fmt.Println(raw)
+		}
+
+		result += raw
+	}
+
+	return result
+}

core/token.go

@@ -0,0 +1,288 @@
+package core
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/fatih/color"
+	"github.com/j3ssie/osmedeus/execution"
+	"github.com/j3ssie/osmedeus/libs"
+	"github.com/j3ssie/osmedeus/utils"
+	"github.com/spf13/viper"
+)
+
+func SetupOSEnv(options *libs.Options) {
+	utils.DebugF("Loading all environment variables from: %v", options.TokenConfigFile)
+	v = viper.New()
+	v.SetConfigName("osm-var")
+	v.SetConfigType("yaml")
+	v.AddConfigPath(path.Dir(options.TokenConfigFile))
+
+	// Read the configuration file
+	err := v.ReadInConfig()
+	if err != nil {
+		v.SetDefault("Storages", map[string]string{
+			// path of secret key for push result
+			// ~/.osmedeus/secret/storages_key
+			"secret_key": utils.GetOSEnv("SECRET_KEY", "SECRET_KEY"),
+			// the repo format should be like this "git@gitlab.com:j3ssie/example.git",
+			"summary_storage":   path.Join(options.Env.RootFolder, "storages/summary"),
+			"summary_repo":      utils.GetOSEnv("SUMMARY_REPO", "SUMMARY_REPO"),
+			"subdomain_storage": path.Join(options.Env.RootFolder, "storages/subdomain"),
+			"subdomain_repo":    utils.GetOSEnv("SUBDOMAIN_REPO", "SUBDOMAIN_REPO"),
+			"assets_storage":    path.Join(options.Env.RootFolder, "storages/assets"),
+			"assets_repo":       utils.GetOSEnv("ASSETS_REPO", "ASSETS_REPO"),
+			"ports_storage":     path.Join(options.Env.RootFolder, "storages/ports"),
+			"ports_repo":        utils.GetOSEnv("PORTS_REPO", "PORTS_REPO"),
+			"http_storage":      path.Join(options.Env.RootFolder, "storages/http"),
+			"http_repo":         utils.GetOSEnv("HTTP_REPO", "HTTP_REPO"),
+			"vuln_storage":      path.Join(options.Env.RootFolder, "storages/vuln"),
+			"vuln_repo":         utils.GetOSEnv("VULN_REPO", "VULN_REPO"),
+			"paths_storage":     path.Join(options.Env.RootFolder, "storages/paths"),
+			"paths_repo":        utils.GetOSEnv("PATHS_REPO", "PATHS_REPO"),
+			"mics_storage":      path.Join(options.Env.RootFolder, "storages/mics"),
+			"mics_repo":         utils.GetOSEnv("MICS_REPO", "MICS_REPO"),
+		})
+
+		// dedicated storages
+		v.SetDefault("Git", map[string]string{
+			"base_url":     utils.GetOSEnv("GITLAB_BASE_URL", "https://gitlab.com"),
+			"api":          utils.GetOSEnv("GITLAB_API_TOKEN", "GITLAB_API_TOKEN"),
+			"username":     utils.GetOSEnv("GITLAB_USER", "GITLAB_USER"),
+			"password":     utils.GetOSEnv("GITLAB_PASS", "GITLAB_PASS"),
+			"group":        utils.GetOSEnv("GITLAB_GROUP", "GITLAB_GROUP"),
+			"prefix_name":  utils.GetOSEnv("GITLAB_PREFIX_NAME", "deosm"),
+			"default_tag":  utils.GetOSEnv("GITLAB_DEFAULT_TAG", "osmd"),
+			"default_user": utils.GetOSEnv("GITLAB_DEFAULT_USER", "j3ssie"),
+			"default_uid":  utils.GetOSEnv("GITLAB_DEFAULT_UID", "3537075"),
+			"destorage":    path.Join(options.Env.RootFolder, "destorage"),
+		})
+
+		v.SetDefault("Notification", map[string]string{
+			"client_name":                utils.GetOSEnv("CLIENT_NAME", "CLIENT_NAME"),
+			"slack_status_channel":       utils.GetOSEnv("SLACK_STATUS_CHANNEL", "SLACK_STATUS_CHANNEL"),
+			"slack_report_channel":       utils.GetOSEnv("SLACK_REPORT_CHANNEL", "SLACK_REPORT_CHANNEL"),
+			"slack_diff_channel":         utils.GetOSEnv("SLACK_DIFF_CHANNEL", "SLACK_DIFF_CHANNEL"),
+			"slack_webhook":              utils.GetOSEnv("SLACK_WEBHOOK", "SLACK_WEBHOOK"),
+			"telegram_channel":           utils.GetOSEnv("TELEGRAM_CHANNEL", "TELEGRAM_CHANNEL"),
+			"telegram_status_channel":    utils.GetOSEnv("TELEGRAM_STATUS_CHANNEL", "TELEGRAM_STATUS_CHANNEL"),
+			"telegram_report_channel":    utils.GetOSEnv("TELEGRAM_REPORT_CHANNEL", "TELEGRAM_REPORT_CHANNEL"),
+			"telegram_sensitive_channel": utils.GetOSEnv("TELEGRAM_SENSITIVE_CHANNEL", "TELEGRAM_SENSITIVE_CHANNEL"),
+			"telegram_dirb_channel":      utils.GetOSEnv("TELEGRAM_DIRB_CHANNEL", "TELEGRAM_DIRB_CHANNEL"),
+			"telegram_mics_channel":      utils.GetOSEnv("TELEGRAM_MICS_CHANNEL", "TELEGRAM_MICS_CHANNEL"),
+		})
+
+		v.SetDefault("Cdn", map[string]string{
+			"cdn_s3_bucket":      utils.GetOSEnv("CDN_S3_BUCKET", "CDN_S3_BUCKET"),
+			"cdn_aws_access_key": utils.GetOSEnv("CDN_AWS_ACCESS_KEY", "CDN_AWS_ACCESS_KEY"),
+			"cdn_aws_secret_key": utils.GetOSEnv("CDN_AWS_SECRET_KEY", "CDN_AWS_SECRET_KEY"),
+			"cdn_aws_region":     utils.GetOSEnv("CDN_AWS_REGION", "ap-southeast-1"),
+		})
+
+		// default tokens but you're feel free to add more and it will be automatically loaded to ENV
+		v.SetDefault("Tokens", map[string]string{
+			"slack":    utils.GetOSEnv("SLACK_API_TOKEN", "SLACK_API_TOKEN"),
+			"telegram": utils.GetOSEnv("TELEGRAM_API_TOKEN", "TELEGRAM_API_TOKEN"),
+			"gitlab":   utils.GetOSEnv("GITLAB_API_TOKEN", "GITLAB_API_TOKEN"),
+			"github":   utils.GetOSEnv("GITHUB_API_KEY", "GITHUB_API_KEY"),
+		})
+
+		if ok := v.WriteConfigAs(options.TokenConfigFile); ok != nil {
+			utils.ErrorF("Error writing config file: %s", ok)
+		}
+		utils.InforF("Created a new token configuration file at %s", color.HiCyanString(options.ConfigFile))
+
+	}
+
+	if err := v.ReadInConfig(); err != nil {
+		utils.ErrorF("Error reading config file, %s", err)
+	}
+	// Read tokens as a list of maps
+	tokens := v.GetStringMapString("tokens")
+	if len(tokens) > 0 {
+		utils.DebugF("Adding %v tokens to the environment variables", color.HiMagentaString("%v", len(tokens)))
+		// Iterate through each token
+		for name, value := range tokens {
+			// automatic convert to upper case
+			name = strings.ToUpper(name)
+
+			// skip if the value is equal to the name
+			if name == value {
+				continue
+			}
+
+			utils.DebugF("Setting environment variable: %v -- %v", name, value)
+
+			err := os.Setenv(name, value)
+			if err != nil {
+				utils.ErrorF("Error setting environment variable: %v -- %v", name, err)
+			}
+		}
+	}
+	// get all the config that need to be set manually
+	GetStorages(options)
+	GetStorages(options)
+	GetNotification(options)
+	GetGit(options)
+	GetCdn(options)
+}
+
+func GetStorages(options *libs.Options) {
+	if !options.PremiumPackage || utils.GetOSEnv("ENABLE_GIT_STORAGES", "") != "TRUE" {
+		return
+	}
+
+	storages := v.GetStringMapString("Storages")
+
+	// get variables from config.yaml
+	storagesOptions := make(map[string]string)
+	for k, dest := range storages {
+		storages[k] = utils.NormalizePath(dest)
+		storagesOptions[k] = utils.NormalizePath(dest)
+	}
+	secretKey := storages["secret_key"]
+	if secretKey == "" || secretKey == "SECRET_KEY" {
+		return
+	}
+
+	storagesOptions["secret_key"] = secretKey
+	// load default existing key if it exists
+	defaultKey := path.Join(options.Env.BaseFolder, "secret/storages_key")
+	if !utils.FileExists(secretKey) && utils.FileExists(defaultKey) {
+		utils.InforF("Loaded default secret for storages from: %v", color.HiCyanString(defaultKey))
+		if _, err := utils.RunCommandWithErr(fmt.Sprintf("cp %s %s && chmod 600 %s", defaultKey, secretKey, secretKey)); err != nil {
+			utils.ErrorF("error copying default secret key: %v", defaultKey)
+		}
+	}
+
+	if !utils.FileExists(secretKey) {
+		utils.InforF("No SSH key for storages found. Generate a new one at: %v", color.HiCyanString(secretKey))
+		if _, err := utils.RunCommandWithErr(fmt.Sprintf(`ssh-keygen -t ed25519 -f %s -q -N ''`, secretKey)); err != nil {
+			color.Red("[-] error generated SSH Key for storages at: %v", secretKey)
+			return
+		}
+		utils.InforF("Please add the public key at %v to your gitlab profile", color.HiCyanString(secretKey+".pub"))
+	}
+
+	if !utils.FileExists("~/.gitconfig") {
+		utils.WarnF("Looks like you didn't set up the git user at %v yet", color.HiCyanString("~/.gitconfig"))
+		utils.WarnF("💡 Init git info with this command: %s", color.HiCyanString(`git config --global user.name "your-username" && git config --global user.email "your-username@users.noreply.gitlab.com"`))
+	}
+
+	if options.CustomGit {
+		// in case custom repo is set
+		for _, env := range os.Environ() {
+			// the ENV should be OSM_SUMMARY_STORAGE, OSM_SUMMARY_REPO
+			if strings.HasSuffix(env, "OSM_") {
+				data := strings.Split(env, "=")
+				key := strings.ToLower(data[0])
+				value := strings.Replace(env, data[0]+"=", "", -1)
+
+				if strings.HasSuffix(key, "summary_storage") {
+					storagesOptions["summary_storage"] = value
+				}
+				if strings.HasSuffix(key, "summary_repo") {
+					storagesOptions["summary_repo"] = value
+				}
+
+				if strings.HasSuffix(key, "assets_storage") {
+					storagesOptions["assets_storage"] = value
+				}
+				if strings.HasSuffix(key, "assets_repo") {
+					storagesOptions["assets_repo"] = value
+				}
+
+				if strings.HasSuffix(key, "ports_storage") {
+					storagesOptions["ports_storage"] = value
+				}
+				if strings.HasSuffix(key, "ports_repo") {
+					storagesOptions["ports_repo"] = value
+				}
+			}
+		}
+	}
+
+	storagesOptions[storages["summary_storage"]] = storages["summary_repo"]
+	storagesOptions[storages["subdomain_storage"]] = storages["subdomain_repo"]
+	storagesOptions[storages["http_storage"]] = storages["http_repo"]
+	storagesOptions[storages["assets_storage"]] = storages["assets_repo"]
+	storagesOptions[storages["mics_storage"]] = storages["mics_repo"]
+	storagesOptions[storages["ports_storage"]] = storages["ports_repo"]
+	storagesOptions[storages["paths_storage"]] = storages["paths_repo"]
+	storagesOptions[storages["vuln_storage"]] = storages["vuln_repo"]
+	options.Storages = storagesOptions
+
+	// disable git feature or no secret key found
+	if options.NoGit {
+		options.Storages["secret_key"] = ""
+	}
+	if options.Storages["secret_key"] == "" {
+		options.NoGit = true
+	}
+
+	if options.NoGit {
+		return
+	}
+	execution.CloneRepo(storages["summary_repo"], storages["summary_storage"], *options)
+	execution.CloneRepo(storages["http_repo"], storages["http_storage"], *options)
+	execution.CloneRepo(storages["assets_repo"], storages["assets_storage"], *options)
+	execution.CloneRepo(storages["subdomain_repo"], storages["subdomain_storage"], *options)
+	execution.CloneRepo(storages["ports_repo"], storages["ports_storage"], *options)
+	execution.CloneRepo(storages["mics_repo"], storages["mics_storage"], *options)
+	execution.CloneRepo(storages["paths_repo"], storages["paths_storage"], *options)
+	execution.CloneRepo(storages["vuln_repo"], storages["vuln_storage"], *options)
+}
+
+// GetNotification get storge repos
+func GetNotification(options *libs.Options) {
+	noti := v.GetStringMapString("Notification")
+	tokens := v.GetStringMapString("Tokens")
+
+	// tokens
+	options.Noti.SlackToken = tokens["slack"]
+	options.Noti.TelegramToken = tokens["telegram"]
+
+	// this mean you're not setup the notification yet
+	if len(options.Noti.TelegramToken) < 20 {
+		options.NoNoti = true
+	}
+
+	options.Noti.ClientName = noti["client_name"]
+
+	options.Noti.SlackStatusChannel = noti["slack_status_channel"]
+	options.Noti.SlackReportChannel = noti["slack_report_channel"]
+	options.Noti.SlackDiffChannel = noti["slack_diff_channel"]
+	options.Noti.SlackWebHook = noti["slack_webhook"]
+	options.Noti.TelegramChannel = noti["telegram_channel"]
+	options.Noti.TelegramSensitiveChannel = noti["telegram_sensitive_channel"]
+	options.Noti.TelegramReportChannel = noti["telegram_report_channel"]
+	options.Noti.TelegramStatusChannel = noti["telegram_status_channel"]
+	options.Noti.TelegramDirbChannel = noti["telegram_dirb_channel"]
+	options.Noti.TelegramMicsChannel = noti["telegram_mics_channel"]
+}
+
+// GetCdn get options for client
+func GetCdn(options *libs.Options) {
+	cdn := v.GetStringMapString("Cdn")
+	options.Cdn.Bucket = cdn["cdn_s3_bucket"]
+	options.Cdn.AccessKeyId = cdn["cdn_aws_access_key"]
+	options.Cdn.SecretKey = cdn["cdn_aws_secret_key"]
+	options.Cdn.Region = cdn["cdn_aws_region"]
+}
+
+// GetGit get options for client
+func GetGit(options *libs.Options) {
+	git := v.GetStringMapString("Git")
+	options.Git.BaseURL = git["base_url"]
+	options.Git.DeStorage = git["destorage"]
+	options.Git.Token = git["api"]
+	options.Git.Username = git["username"]
+	options.Git.Password = git["password"]
+	options.Git.Group = git["group"]
+	options.Git.DefaultPrefix = git["prefix_name"]
+	options.Git.DefaultTag = git["default_tag"]
+	options.Git.DefaultUser = git["default_user"]
+	options.Git.DefaultUID = utils.StrToInt(git["default_uid"])
+}

core/update.go

@@ -0,0 +1,336 @@
+package core
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/cenkalti/backoff/v4"
+	"github.com/fatih/color"
+	"github.com/hashicorp/go-version"
+	"github.com/j3ssie/osmedeus/libs"
+	"github.com/j3ssie/osmedeus/provider"
+	"github.com/j3ssie/osmedeus/utils"
+	jsoniter "github.com/json-iterator/go"
+	"github.com/mitchellh/go-homedir"
+)
+
+/* Mostly calling OS commands for double-check the PATH too */
+
+func CheckUpdate(options *libs.Options) bool {
+	// t.Format("02-Jan-2006")
+
+	var shouldUpdate bool
+	// ~/.osmedeus/update/metadata.json
+	options.Update.UpdateConfig = path.Join(utils.NormalizePath(options.Env.RootFolder), "update")
+	utils.MakeDir(options.Update.UpdateConfig)
+
+	if options.Update.MetaDataURL == "" {
+		options.Update.MetaDataURL = fmt.Sprintf("%s/public.json", libs.METADATA)
+		if options.PremiumPackage {
+			options.Update.MetaDataURL = fmt.Sprintf("%s/premium.json", libs.METADATA)
+		}
+	}
+
+	utils.InforF("Checking metadata information from: %v", options.Update.MetaDataURL)
+	metadataFile := path.Join(options.Update.UpdateConfig, "metadata.json")
+
+	var oldMetaData libs.UpdateMetaData
+	oldMetaData.CoreVersion = libs.VERSION
+	oldMetaData.WorkflowVersion = "v0.0.1"
+
+	if utils.FileExists(metadataFile) {
+		oldMetaDataContent := utils.GetFileContent(metadataFile)
+		if err := jsoniter.UnmarshalFromString(oldMetaDataContent, &oldMetaData); err != nil {
+			utils.ErrorF("error to parse metadata: %v", metadataFile)
+			return false
+		}
+	}
+
+	var newMetaData libs.UpdateMetaData
+	res := utils.SendGET("", options.Update.MetaDataURL)
+	if res.StatusCode == 200 {
+		if err := jsoniter.UnmarshalFromString(res.Body, &newMetaData); err != nil {
+			utils.ErrorF("error to parse metadata: %v", options.Update.MetaDataURL)
+			return false
+		}
+
+		utils.InforF("Writing metadata to: %v", color.HiCyanString(metadataFile))
+		if data, err := jsoniter.MarshalToString(&newMetaData); err == nil {
+			utils.WriteToFile(metadataFile, data)
+		}
+	} else {
+		utils.ErrorF("error fetching metadata from: %v", options.Update.MetaDataURL)
+		return false
+	}
+
+	utils.DebugF(res.Body)
+
+	v1, err := version.NewVersion(oldMetaData.CoreVersion)
+	if err != nil {
+		utils.ErrorF("error parsing version: %v -- %v", oldMetaData.CoreVersion, err)
+		return false
+	}
+
+	// get from metadata URL
+	v2, err := version.NewVersion(newMetaData.CoreVersion)
+	if err != nil {
+		utils.ErrorF("error parsing version: %v -- %v", newMetaData.CoreVersion, err)
+
+		return false
+	}
+
+	// Comparison example. There is also GreaterThan, Equal, and just
+	// a simple Compare that returns an int allowing easy >=, <=, etc.
+	if v1.LessThan(v2) {
+		fmt.Printf("Your current %s %s are outdated. Latest is %s\n", libs.BINARY, color.HiMagentaString("%v", v1), color.HiGreenString("%v", v2))
+		shouldUpdate = true
+	} else {
+		fmt.Printf("You're using %s core latest version %s updated at %s\n", libs.BINARY, color.HiMagentaString("%v", v1), color.HiGreenString("%v", newMetaData.UpdatedAt))
+	}
+
+	// check workflow version if core is updated
+	if !shouldUpdate {
+		wfv1, err := version.NewVersion(oldMetaData.WorkflowVersion)
+		if err != nil {
+			utils.ErrorF("error parsing version: %v -- %v", oldMetaData.WorkflowVersion, err)
+		}
+		wfv2, err := version.NewVersion(newMetaData.WorkflowVersion)
+		if err != nil {
+			utils.ErrorF("error parsing version: %v -- %v", newMetaData.WorkflowVersion, err)
+		}
+		if wfv1.LessThan(wfv2) {
+			fmt.Printf("Your current %s workflow %s are outdated. Latest is %s\n", libs.BINARY, color.HiMagentaString("%v", v1), color.HiGreenString("%v", v2))
+			shouldUpdate = true
+		}
+	}
+
+	if shouldUpdate {
+		home, _ := homedir.Dir()
+		fmt.Printf("📖 Run %s again to update Check out this page for more detail: %s\n", color.HiGreenString("the same install script"), color.HiGreenString("https://docs.osmedeus.org/installation/"))
+		fmt.Printf("💡 If you want a fresh install please run the command: %s\n", color.HiBlueString("rm -rf %s/osmedeus-base %s/.osmedeus", home, home))
+	}
+
+	return shouldUpdate
+}
+
+func GetUpdateURL(options libs.Options) string {
+	if !options.PremiumPackage {
+		utils.InforF("Getting update url of public community package")
+		return libs.INSTALL
+	}
+	utils.InforF("💎 Getting update url of premium package")
+
+	providerConfigs, err := provider.ParseProvider(options.CloudConfigFile)
+	if err != nil {
+		utils.ErrorF("error to parse provider config: %v", err)
+		return ""
+	}
+
+	return providerConfigs.Builder.BuildRepo
+}
+
+func RunUpdate(options libs.Options) error {
+	if options.Update.UpdateURL == "" {
+		return fmt.Errorf("no update URL")
+	}
+
+	if options.Update.CleanOldData {
+		utils.InforF("Cleaning old data: %s", color.HiCyanString(options.Env.BaseFolder))
+		os.RemoveAll(options.Env.BaseFolder)
+	}
+
+	if options.PremiumPackage {
+		utils.InforF("Running update from premium package install script")
+	} else {
+		utils.InforF("Running update from: %v", color.HiCyanString(options.Update.UpdateURL))
+	}
+
+	options.Update.UpdateScript = fmt.Sprintf("/tmp/%s-update.sh", libs.BINARY)
+	cmd := fmt.Sprintf("rm -rf %s && wget -qO %s %s ", options.Update.UpdateScript, options.Update.UpdateScript, options.Update.UpdateURL)
+	_, err := utils.RunCommandWithErr(cmd)
+	if err != nil {
+		utils.ErrorF("error to run update script: %v", err)
+		return err
+	}
+
+	if !utils.FileExists(options.Update.UpdateScript) {
+		utils.ErrorF("update script doesn't exist: %v", options.Update.UpdateScript)
+		return fmt.Errorf("update script doesn't exist: %v", options.Update.UpdateScript)
+	}
+
+	if _, err := utils.RunCommandSteamOutput(fmt.Sprintf(`bash %s`, options.Update.UpdateScript)); err != nil {
+		utils.ErrorF("error to run update script: %v", err)
+		return err
+	}
+	return nil
+}
+
+func GenerateMetaData(options libs.Options) {
+	utils.InforF("Generating metadata to: %v", color.HiCyanString(options.Update.GenerateMeta))
+	t := time.Now()
+	t.Format("2006-01-02T15:04:05")
+	var updateData = libs.UpdateMetaData{
+		CoreVersion:     libs.VERSION,
+		WorkflowVersion: libs.VERSION,
+		UpdatedAt:       t.Format("2006-01-02T15:04"),
+	}
+
+	if data, ok := jsoniter.MarshalToString(updateData); ok == nil {
+		utils.InforF("Generate meta data: %v", data)
+		utils.WriteToFile(options.Update.GenerateMeta, data)
+	}
+}
+
+func GitUpdate(opt libs.Options) error {
+	cmd := fmt.Sprintf("GIT_SSH_COMMAND='ssh -o StrictHostKeyChecking=no -i %v' git clone --depth=1 %v %v", opt.Storages["secret_key"], opt.Update.UpdateURL, opt.Update.UpdateFolder)
+	_, err := utils.RunCommandWithErr(cmd)
+	return err
+}
+
+func HTTPUpdate(opt libs.Options) error {
+	cmd := fmt.Sprintf("wget -q %s -O %s", opt.Update.UpdateURL, opt.Update.UpdateFolder)
+	_, err := utils.RunCommandWithErr(cmd)
+	return err
+}
+
+func DownloadUpdate(opt libs.Options) error {
+	os.RemoveAll(opt.Update.UpdateFolder)
+	utils.InforF("Downloading the update folder via %v: %v", opt.Update.UpdateType, opt.Update.UpdateURL)
+	var err error
+
+	backOff := backoff.NewExponentialBackOff()
+	backOff.MaxElapsedTime = 1200 * time.Second
+	backOff.Multiplier = 2.0
+	backOff.InitialInterval = 30 * time.Second
+
+	operation := func() error {
+		switch strings.ToLower(opt.Update.UpdateType) {
+		case "git":
+			err = GitUpdate(opt)
+		case "s3", "http":
+			err = HTTPUpdate(opt)
+		default:
+			err = GitUpdate(opt)
+		}
+		if err != nil {
+			utils.ErrorF("error downloading update content: %s -- %s", opt.Update.UpdateType, opt.Update.UpdateURL)
+		}
+		return err
+	}
+	err = backoff.Retry(operation, backOff)
+	if err != nil {
+		utils.ErrorF("error downloading update content: %s -- %s", opt.Update.UpdateType, opt.Update.UpdateURL)
+		return err
+	}
+	return nil
+}
+
+func Update(opt libs.Options) {
+	os.RemoveAll(opt.Update.UpdateFolder)
+	utils.MakeDir(opt.Update.UpdateFolder)
+
+	updateScript := fmt.Sprintf("%s/update.sh", opt.Update.UpdateFolder)
+	cmd := fmt.Sprintf("wget -q %s -O %s/install.sh", opt.Update.UpdateURL, updateScript)
+	if _, err := utils.RunCommandWithErr(cmd); err != nil {
+		utils.ErrorF("error downloading the update script: %v", opt.Update.UpdateURL)
+		return
+	}
+
+	cmd = fmt.Sprintf("bash %s", updateScript)
+	if _, err := utils.RunCommandWithErr(cmd); err != nil {
+		utils.ErrorF("error running the update script: %v", updateScript)
+		return
+	}
+}
+
+func UpdateVuln(opt libs.Options) {
+	utils.InforF("Updating Vulnerability Database only")
+
+	// update nuclei templates
+	utils.DebugF("Updating Nuclei Templates")
+	nucleiTemplate := utils.NormalizePath("~/nuclei-templates")
+
+	if utils.DirLength(nucleiTemplate) > 0 {
+		os.RemoveAll(nucleiTemplate)
+	}
+	utils.RunOSCommand(fmt.Sprintf("git clone --depth=1 https://github.com/projectdiscovery/nuclei-templates.git %s", nucleiTemplate))
+}
+
+func UpdateBase(opt libs.Options) {
+	err := DownloadUpdate(opt)
+	if err != nil {
+		return
+	}
+
+	// change the folder since we will update it
+	if opt.Update.IsUpdateBin {
+		utils.InforF("Updating External binaries")
+		binPath := path.Join(opt.Update.UpdateFolder, "binaries")
+		utils.Move(binPath, opt.Env.BinariesFolder)
+		opt.Update.UpdateFolder = path.Join(opt.Update.UpdateFolder, fmt.Sprintf("%s-base", libs.BINARY))
+	}
+
+	// update Env
+	utils.InforF("Updating Environments Data")
+	utils.Move(path.Join(opt.Update.UpdateFolder, "data"), opt.Env.DataFolder)
+	utils.Move(path.Join(opt.Update.UpdateFolder, "workflow"), opt.Env.WorkFlowsFolder)
+	utils.Move(path.Join(opt.Update.UpdateFolder, "ose"), opt.Env.OseFolder)
+	utils.Move(path.Join(opt.Update.UpdateFolder, "ui"), opt.Env.UIFolder)
+
+	utils.Move(path.Join(opt.Update.UpdateFolder, "clouds"), opt.Env.CloudConfigFolder)
+	os.Chmod(opt.Cloud.SecretKey, 0600)
+
+	// update osmedeus core binary
+	corePath, err := exec.LookPath(libs.BINARY)
+	utils.InforF("Updating %v binary at %v", color.HiCyanString(libs.BINARY), color.HiCyanString(corePath))
+	if err == nil {
+		os.RemoveAll(corePath)
+		newBin := fmt.Sprintf("%s/dist/%s", strings.TrimRight(opt.Update.UpdateFolder, "/"), libs.BINARY)
+		unZipCmd := fmt.Sprintf("unzip %s/dist/%s-linux.zip -d %s/dist/", strings.TrimRight(opt.Update.UpdateFolder, "/"), libs.BINARY, strings.TrimRight(opt.Update.UpdateFolder, "/"))
+		utils.RunOSCommand(unZipCmd)
+
+		// update binaries in gopath
+		goPath := utils.GetOSEnv("GOPATH", "GOPATH")
+		if goPath != "GOPATH" {
+			goPath = path.Join(goPath, fmt.Sprintf("bin/%s", libs.BINARY))
+			os.RemoveAll(goPath)
+			utils.RunOSCommand(fmt.Sprintf("cp %s %s", newBin, goPath))
+
+			// go path but in plugins folder
+			goPath = path.Join(opt.Env.BinariesFolder, "go", libs.BINARY)
+			os.RemoveAll(goPath)
+			utils.RunOSCommand(fmt.Sprintf("cp %s %s", newBin, goPath))
+		}
+		utils.Move(newBin, corePath)
+	}
+
+	// update vulnerability signatures
+	utils.InforF("Updating Jaeles Signatures")
+	jaelesSign := path.Join(opt.Update.UpdateFolder, "pro-signatures")
+	if utils.DirLength(jaelesSign) > 0 {
+		utils.RunOSCommand(fmt.Sprintf("jaeles config reload --signDir %s", jaelesSign))
+		utils.Move(jaelesSign, "~/pro-signatures")
+	} else {
+		os.RemoveAll(utils.NormalizePath("~/pro-signatures"))
+		utils.RunOSCommand(fmt.Sprintf("GIT_SSH_COMMAND='ssh -o StrictHostKeyChecking=no -i %s' git clone --depth=1 git@gitlab.com:j3ssie/pro-signatures ~/pro-signatures", opt.Storages["secret_key"]))
+		utils.RunOSCommand(fmt.Sprintf("rm -rf ~/custom-nuclei-template && GIT_SSH_COMMAND='ssh -o StrictHostKeyChecking=no -i %s' git clone --depth=1 git@gitlab.com:j3ssie/custom-nuclei-template.git ~/custom-nuclei-template", opt.Storages["secret_key"]))
+		utils.RunOSCommand("jaeles config reload --signDir ~/pro-signatures")
+	}
+
+	// update nuclei templates
+	utils.InforF("Updating Nuclei Templates")
+	nucleiTemplate := path.Join(opt.Update.UpdateFolder, "nuclei-templates")
+	if utils.DirLength(nucleiTemplate) > 0 {
+		utils.Move(nucleiTemplate, utils.NormalizePath("~/nuclei-templates"))
+	} else {
+		utils.RunOSCommand(fmt.Sprintf("git clone --depth=1 https://github.com/projectdiscovery/nuclei-templates.git ~/nuclei-templates"))
+	}
+
+	// clean up
+	utils.InforF("Clean up update folder")
+	os.RemoveAll(opt.Update.UpdateFolder)
+}

core/validate.go

@@ -0,0 +1,144 @@
+package core
+
+import (
+	"fmt"
+	"path"
+	"strings"
+
+	"github.com/fatih/color"
+	"github.com/go-playground/validator/v10"
+	"github.com/j3ssie/osmedeus/libs"
+	"github.com/j3ssie/osmedeus/utils"
+)
+
+func (r *Runner) Validator() error {
+	if r.RequiredInput == "" || r.Opt.DisableValidateInput {
+		return nil
+	}
+
+	r.RequiredInput = strings.ToLower(strings.TrimSpace(r.RequiredInput))
+	var inputAsFile bool
+	// cidr, cidr-file
+	if strings.HasSuffix(r.RequiredInput, "-file") || r.RequiredInput == "file" {
+		inputAsFile = true
+	}
+	v := validator.New()
+
+	// if input as a file
+	if utils.FileExists(r.Input) && inputAsFile {
+		r.InputType = "file"
+		inputs := utils.ReadingLines(r.Input)
+
+		for index, input := range inputs {
+			if strings.TrimSpace(input) == "" {
+				continue
+			}
+			// no more validation if it's file 'validator: file'
+			if r.RequiredInput == "file" {
+				continue
+			}
+
+			inputType, err := validate(v, input)
+			// fmt.Println("r.RequiredInput, inputType", r.RequiredInput, inputType)
+			if err == nil {
+				// really validate thing
+				if !strings.HasPrefix(r.RequiredInput, inputType) {
+					utils.DebugF("validate: %v -- %v", input, inputType)
+					errString := fmt.Sprintf("line %v in %v file not match the require input: %v -- %v", index, r.Input, input, inputType)
+					utils.ErrorF(errString)
+					return fmt.Errorf(errString)
+				}
+			}
+		}
+		return nil
+
+	}
+
+	var err error
+	r.InputType, err = validate(v, r.Input)
+	if err != nil {
+		utils.ErrorF("unrecognized input: %v", r.Input)
+		return err
+	}
+	utils.InforF("Start validating input: %v -- %v", color.HiCyanString(r.Input), color.HiCyanString(r.InputType))
+
+	if !strings.HasPrefix(r.RequiredInput, r.InputType) {
+		return fmt.Errorf("input does not match the require validation: inputType:%v -- requireType:%v", r.InputType, r.RequiredInput)
+	}
+
+	if inputAsFile {
+		utils.MakeDir(libs.TEMP)
+		suffix := utils.RandomString(4)
+		if r.Opt.Scan.SuffixName != "" {
+			suffix = r.Opt.Scan.SuffixName + "-" + utils.RandomString(4)
+		}
+		dest := path.Join(libs.TEMP, fmt.Sprintf("%v-%v", utils.StripPath(r.Input), suffix))
+		if r.Opt.Scan.CustomWorkspace != "" {
+			dest = path.Join(libs.TEMP, fmt.Sprintf("%v-%v", utils.StripPath(r.Opt.Scan.CustomWorkspace), suffix))
+		}
+		utils.WriteToFile(dest, r.Input)
+		utils.InforF("Convert input to a file: %v", dest)
+		r.Input = dest
+		r.Target = ParseInput(r.Input, r.Opt)
+	}
+
+	utils.DebugF("validator: input:%v -- type: %v -- require:%v", r.Input, r.InputType, r.RequiredInput)
+	return nil
+}
+
+func validate(v *validator.Validate, raw string) (string, error) {
+	var err error
+	var inputType string
+
+	if utils.FileExists(raw) {
+		inputType = "file"
+	}
+
+	err = v.Var(raw, "required,url")
+	if err == nil {
+		inputType = "url"
+	}
+
+	err = v.Var(raw, "required,ipv4")
+	if err == nil {
+		inputType = "ip"
+	}
+
+	err = v.Var(raw, "required,fqdn")
+	if err == nil {
+		inputType = "domain"
+	}
+
+	err = v.Var(raw, "required,hostname")
+	if err == nil {
+		inputType = "domain"
+	}
+
+	err = v.Var(raw, "required,cidr")
+	if err == nil {
+		inputType = "cidr"
+	}
+
+	err = v.Var(raw, "required,uri")
+	if err == nil {
+		inputType = "url"
+	}
+
+	err = v.Var(raw, "required,uri")
+	if err == nil {
+		inputType = "url"
+		if strings.HasPrefix(raw, "https://github.com") || strings.HasPrefix(raw, "https://gitlab.com") {
+			inputType = "git-url"
+		}
+	}
+
+	if strings.HasPrefix(raw, "git@") {
+		inputType = "git-url"
+	}
+
+	if inputType == "" {
+		return "", fmt.Errorf("unrecognized input")
+	}
+
+	return inputType, nil
+}

core/validate_test.go

@@ -0,0 +1,53 @@
+package core
+
+import (
+	"fmt"
+	"github.com/j3ssie/osmedeus/libs"
+	"testing"
+)
+
+func TestValidator(t *testing.T) {
+	input := "target.com"
+	opt := libs.Options{}
+	runner, _ := InitRunner(input, opt)
+	runner.RequiredInput = "domain"
+	runner.Validator()
+
+	runner.Input = "apple.com"
+	runner.Validator()
+
+	fmt.Printf("runner.InputType --> %v:%v -- %s\n", runner.RequiredInput, runner.InputType, runner.Input)
+
+	runner.Input = "1.2.3.4"
+	runner.Validator()
+
+	runner.Input = "http://127.0.0.1/q"
+	runner.Validator()
+	runner.Input = "sub.domain.com"
+	runner.Validator()
+
+	runner.Input = "1.2.3.4/24"
+	runner.Validator()
+
+	runner.Input = "https://github.com/j3ssie/osmedeus"
+	runner.Validator()
+	fmt.Printf("==> runner.InputType --> %v:%v -- %s\n\n", runner.RequiredInput, runner.InputType, runner.Input)
+
+	runner.Input = "git@github.com:j3ssie/osmedeus.git"
+	runner.Validator()
+	fmt.Printf("==> runner.InputType --> %v:%v -- %s\n\n", runner.RequiredInput, runner.InputType, runner.Input)
+
+	//
+	////raw := "tcp://git@gitlab.com:j3ssie/osmd-assets"
+	//raw := "git@gitlab.com/j3ssie/osmd-assets"
+	//v := validator.New()
+	//err := v.Var(raw, "required,uri")
+	//fmt.Println(err)
+	//
+	//err = v.Var(raw, "required,datauri")
+	//fmt.Println(err)
+
+	if runner.InputType == "" {
+		t.Errorf("Error Validator")
+	}
+}

database/connect.go

@@ -0,0 +1,63 @@
+package database
+
+// load driver
+
+// // DB gorm connector
+// var DB *gorm.DB
+
+// // InitDB connect to db
+// func InitDB(options libs.Options) (*gorm.DB, error) {
+// 	newLogger := logger.New(
+// 		log.New(ioutil.Discard, "\r\n", log.LstdFlags), // io writer
+// 		logger.Config{
+// 			SlowThreshold:             time.Second, // Slow SQL threshold
+// 			LogLevel:                  logger.Info, // Log level
+// 			IgnoreRecordNotFoundError: true,        // Ignore ErrRecordNotFound error for logger
+// 			Colorful:                  false,       // Disable color
+// 		},
+// 	)
+
+// 	config := gorm.Config{
+// 		SkipDefaultTransaction:                   false,
+// 		NamingStrategy:                           nil,
+// 		FullSaveAssociations:                     false,
+// 		Logger:                                   newLogger,
+// 		NowFunc:                                  nil,
+// 		DryRun:                                   false,
+// 		PrepareStmt:                              false,
+// 		DisableAutomaticPing:                     false,
+// 		DisableForeignKeyConstraintWhenMigrating: false,
+// 		DisableNestedTransaction:                 false,
+// 		AllowGlobalUpdate:                        false,
+// 		QueryFields:                              false,
+// 		CreateBatchSize:                          0,
+// 		ClauseBuilders:                           nil,
+// 		ConnPool:                                 nil,
+// 		Dialector:                                nil,
+// 		Plugins:                                  nil,
+// 	}
+
+// 	var err error
+// 	if options.Server.DBType == "mysql" {
+// 		utils.InforF("Connect Database at %v:%v", options.Server.DBHost, options.Server.DBPort)
+// 		utils.InforF("Use Database: %v", options.Server.DBName)
+// 		DB, err = gorm.Open(mysql.Open(options.Server.DBConnection), &config)
+// 	} else {
+// 		DB, err = gorm.Open(sqlite.Open(options.Server.DBPath), &config)
+// 	}
+
+// 	if err != nil {
+// 		fmt.Printf("Error Database connect at %v:%v -- %v\n", options.Server.DBHost, options.Server.DBPort, err)
+// 		return nil, err
+// 	}
+
+// 	// scanning data
+// 	// DB.AutoMigrate(&User{})
+// 	//DB.AutoMigrate(&Org{})
+// 	DB.AutoMigrate(&Target{})
+// 	DB.AutoMigrate(&Scan{})
+// 	DB.AutoMigrate(&Report{})
+// 	DB.AutoMigrate(&CloudInstance{})
+// 	// asset data
+// 	return DB, nil
+// }

database/models.go

@@ -0,0 +1,99 @@
+package database
+
+import (
+	"time"
+)
+
+type Model struct {
+	CreatedAt time.Time `json:"created_at,omitempty"`
+	UpdatedAt time.Time `json:"updated_at,omitempty"`
+}
+
+// Schedule store task to do every single time
+type Schedule struct {
+	Model
+	TaskName       string `gorm:"type:varchar(255)" json:"task_name"`
+	RefreshSeconds string `gorm:"type:varchar(255)" json:"refresh_seconds"`
+	Command        string `gorm:"type:longtext" json:"command"`
+	Status         string `gorm:"type:varchar(255)" json:"status"`
+}
+
+///////////
+
+type Scan struct {
+	Model
+
+	// input part
+	InputName string `gorm:"type:varchar(255);not null" json:"input_name"`
+	InputType string `gorm:"type:varchar(255);default:'general'" json:"input_type"`
+
+	TaskName string `gorm:"type:varchar(255)" json:"task_name"`
+	TaskPath string `gorm:"type:varchar(255)" json:"task_path"`
+	TaskType string `gorm:"type:varchar(255);default:'flow'" json:"task_type"`
+
+	MarkDownSunmmary string `gorm:"type:varchar(255)" json:"markdown_summary"`
+	MarkDownReport   string `gorm:"type:varchar(255)" json:"markdown_report"`
+
+	RunningTime   int    `json:"running_time"` // as seconds
+	CurrentModule string `gorm:"type:varchar(255)" json:"current_module"`
+	DoneStep      int    `json:"done_step"`
+	TotalSteps    int    `json:"total_steps"`
+
+	// mics part
+	LogFile   string `json:"log_file"`
+	ProcessID int    `json:"process_id"`
+
+	// progress checking
+	IsRunning bool `json:"is_running"`
+	IsDone    bool `json:"is_done"`
+	IsNew     bool `json:"is_new"`
+	IsError   bool `json:"is_error"`
+	IsStarted bool `json:"is_started"`
+
+	// if the task is running by cloud provider
+	IsPrepared bool   `json:"is_prepared"`
+	IsCloud    bool   `json:"is_cloud"`
+	CloudInfo  string `json:"cloud_info"`
+
+	Target Target `json:"target"`
+}
+
+// runtime object
+type Target struct {
+	InputName string `gorm:"type:varchar(255);unique;not null" json:"input_name"`
+	// @NOTE: below field shouldn't be show in UI
+	// Workspace == InputName but strip out '/'
+	Workspace string `gorm:"type:varchar(255);unique;not null" json:"workspace"`
+	InputType string `gorm:"type:varchar(255);default:'N/A'" json:"input_type"`
+
+	// total number for stat
+	TotalAssets        int `json:"total_assets"`
+	TotalDns           int `json:"total_dns"`
+	TotalTech          int `json:"total_tech"`
+	TotalScreenShot    int `json:"total_screenshot"`
+	TotalVulnerability int `json:"total_vulnerability"`
+	TotalDirb          int `json:"total_dirb"`
+	TotalLink          int `json:"total_link"`
+	TotalArchive       int `json:"total_archive"`
+	TotalIPRange       int `json:"total_ip_range"`
+	TotalCloud         int `json:"total_cloud"`
+	TotalCred          int `json:"total_cred"`
+
+	// flag information
+	IsNew      bool `json:"is_new"`
+	IsWildCard bool `json:"is_wildcard"`
+
+	Reports []Report `json:"reports"`
+}
+
+// Report store reports file record
+type Report struct {
+	ReportName string `gorm:"type:varchar(255)" json:"report_name"`
+	ReportPath string `gorm:"type:longtext" json:"report_path"`
+
+	Module     string `gorm:"type:varchar(255)" json:"module"`
+	ModulePath string `gorm:"type:longtext" json:"module_path"`
+
+	WorkspaceName string `gorm:"type:varchar(255)" json:"workspace_name"`
+	ReportType    string `gorm:"type:varchar(255);default:'text'" json:"report_type"`
+}

database/select.go

@@ -0,0 +1,100 @@
+package database
+
+import (
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"github.com/j3ssie/osmedeus/libs"
+	"github.com/j3ssie/osmedeus/utils"
+	jsoniter "github.com/json-iterator/go"
+)
+
+func GetAllWorkspaces(opt libs.Options) (directories []string) {
+	// Open the specified directory
+	dir, err := os.Open(opt.Env.WorkspacesFolder)
+	if err != nil {
+		return directories
+	}
+	defer dir.Close()
+
+	// Read all entries in the directory
+	entries, err := dir.Readdir(-1)
+	if err != nil {
+		return directories
+	}
+
+	for _, entry := range entries {
+		wsName := entry.Name()
+		directories = append(directories, wsName)
+	}
+
+	return directories
+}
+
+func GetAllScan(opt libs.Options) (scans []Scan) {
+	wss := GetAllWorkspaces(opt)
+	for _, wsName := range wss {
+		runtimeFile := filepath.Join(opt.Env.WorkspacesFolder, wsName, "runtime")
+		if !utils.FileExists(runtimeFile) {
+			continue
+		}
+
+		// parse the content
+		runtimeContent := utils.GetFileContent(runtimeFile)
+		wsData := Scan{}
+		if err := jsoniter.UnmarshalFromString(runtimeContent, &wsData); err == nil {
+
+			// replace the filepath with static prefix
+			wsData.MarkDownReport = strings.ReplaceAll(wsData.MarkDownReport, opt.Env.WorkspacesFolder, path.Join("/", opt.Server.StaticPrefix, "workspaces"))
+			wsData.MarkDownSunmmary = strings.ReplaceAll(wsData.MarkDownSunmmary, opt.Env.WorkspacesFolder, path.Join("/", opt.Server.StaticPrefix, "workspaces"))
+			scans = append(scans, wsData)
+		}
+
+	}
+	return scans
+}
+
+func GetSingleScan(wsName string, opt libs.Options) (scan Scan) {
+	runtimeFile := filepath.Join(opt.Env.WorkspacesFolder, wsName, "runtime")
+	if !utils.FileExists(runtimeFile) {
+		return scan
+	}
+
+	// parse the content
+	runtimeContent := utils.GetFileContent(runtimeFile)
+	if err := jsoniter.UnmarshalFromString(runtimeContent, &scan); err == nil {
+		// replace the filepath with static prefix
+		scan.MarkDownReport = strings.ReplaceAll(scan.MarkDownReport, opt.Env.WorkspacesFolder, path.Join("/", opt.Server.StaticPrefix, "workspaces"))
+		scan.MarkDownSunmmary = strings.ReplaceAll(scan.MarkDownSunmmary, opt.Env.WorkspacesFolder, path.Join("/", opt.Server.StaticPrefix, "workspaces"))
+		return scan
+	}
+	return scan
+}
+
+func GetScanProgress(opt libs.Options) (scans []Scan) {
+	rawScans := GetAllScan(opt)
+
+	for _, scan := range rawScans {
+		scan.Target = Target{}
+		scans = append(scans, scan)
+	}
+
+	return scans
+}
+
+// func GetWorkspaceDetail(wsName string, opt libs.Options) (workspace Scan) {
+// 	runtimeFile := filepath.Join(opt.Env.WorkspacesFolder, wsName, "runtime")
+// 	if !utils.FileExists(runtimeFile) {
+// 		return workspace
+// 	}
+
+// 	// parse the content
+// 	runtimeContent := utils.GetFileContent(runtimeFile)
+// 	if err := jsoniter.UnmarshalFromString(runtimeContent, &workspace); err == nil {
+// 		return workspace
+// 	}
+
+// 	return workspace
+// }

distribute/clean.go

@@ -0,0 +1,54 @@
+package distribute
+
+import (
+	"os"
+	"sync"
+
+	"github.com/fatih/color"
+	"github.com/j3ssie/osmedeus/libs"
+	"github.com/j3ssie/osmedeus/utils"
+	"github.com/jinzhu/copier"
+	"github.com/panjf2000/ants"
+)
+
+func ClearAllInstances(opt libs.Options) {
+	if opt.Cloud.Retry == 0 {
+		opt.Cloud.Retry = 8
+	}
+
+	// select all cloud instance
+	instances := GetAllInstances(opt)
+	if len(instances) == 0 {
+		utils.WarnF("no active cloud instance running")
+		return
+	}
+
+	var wg sync.WaitGroup
+	p, _ := ants.NewPoolWithFunc(opt.Concurrency*5, func(i interface{}) {
+		instance := i.(CloudRunner)
+
+		var options libs.Options
+		copier.Copy(&options, &opt)
+		instance.Opt = opt
+		instance.Provider.IsBackgroundCheck = true
+		instance.Provider.InitClient()
+		instance.Prepare()
+		utils.InforF("Deleting the instance: %v -- %v", instance.Provider.ProviderName, instance.PublicIP)
+		if err := instance.Provider.DeleteInstance(instance.InstanceID); err == nil {
+			utils.InforF("Instance deleted %s -- %s", color.HiYellowString(instance.PublicIP), color.HiYellowString(instance.InstanceID))
+			instanceFile := instance.Opt.Env.InstancesFolder + "/" + instance.InstanceName + "-" + instance.PublicIP + ".json"
+			os.Remove(instanceFile)
+		}
+
+		wg.Done()
+	}, ants.WithPreAlloc(true))
+	defer p.Release()
+
+	utils.InforF("Evaluating the status of %v cloud instances", color.HiMagentaString("%v", len(instances)))
+	for _, instance := range instances {
+		wg.Add(1)
+		_ = p.Invoke(instance)
+	}
+	wg.Wait()
+
+}

distribute/cloud_runner.go

@@ -0,0 +1,201 @@
+package distribute
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"sync"
+
+	"github.com/fatih/color"
+	"github.com/j3ssie/osmedeus/core"
+	"github.com/j3ssie/osmedeus/libs"
+	"github.com/j3ssie/osmedeus/provider"
+	"github.com/j3ssie/osmedeus/utils"
+	"github.com/panjf2000/ants"
+)
+
+type CloudRunner struct {
+	Opt      libs.Options `json:"-"`
+	Provider provider.Provider
+
+	// for storing in local DB
+	TaskName     string `json:"task_name"`
+	TaskType     string `json:"task_type"`
+	Input        string `json:"input"`
+	RawCommand   string `json:"raw_command"`
+	InstanceFile string `json:"instance_file"`
+
+	// core entry point
+	PublicIP      string `json:"public_ip"`
+	DestInstance  string `json:"dest_instance"`
+	SshPublicKey  string `json:"ssh_public_key"`
+	SshPrivateKey string
+	SSHUser       string
+	BasePath      string
+
+	InstanceID   string `json:"instance_id"`
+	InstanceName string `json:"instance_name"`
+	IsError      bool   `json:"is_error"`
+
+	Target map[string]string `json:"-"`
+	Runner core.Runner       `json:"-"`
+}
+
+// InitCloud init cloud runner obj
+func InitCloud(options libs.Options, targets []string) {
+	// init clouds object queue
+	cloudConfigs := GetClouds(options)
+	if options.Cloud.CheckingLimit {
+		return
+	}
+
+	// really start doing something
+	var wg sync.WaitGroup
+	p, _ := ants.NewPoolWithFunc(options.Concurrency, func(i interface{}) {
+		target := i.(string)
+		if strings.TrimSpace(target) == "" {
+			wg.Done()
+			return
+		}
+
+		for {
+			var selectedCloud CloudRunner
+			var isDone bool
+			utils.DebugF("Select available accounts")
+			for _, cloudInfo := range cloudConfigs {
+				// don't run it again for next cloud account
+				if isDone {
+					break
+				}
+				if cloudInfo.Provider.Available {
+					selectedCloud = cloudInfo
+				}
+
+				target = PrepareTarget(target, options)
+				if !options.Cloud.OnlyCreateDroplet {
+					utils.InforF("Initiating the scanning process of %v", color.HiMagentaString(utils.CleanPath(target)))
+				}
+
+				// really start to run scan here
+				err := selectedCloud.Scan(target)
+				if err != nil {
+					utils.ErrorF("error start scan %s", color.HiCyanString(target))
+					if options.Cloud.NoDelete {
+						continue
+					}
+					if ok := selectedCloud.Provider.DeleteInstance(selectedCloud.InstanceID); ok == nil {
+						selectedCloud.DeleteInstanceConfig()
+					}
+					continue
+				}
+
+				if !options.Cloud.OnlyCreateDroplet {
+					utils.InforF("Completed scanning %v", color.HiCyanString(utils.CleanPath(target)))
+					utils.InforF("--------------------------------")
+				}
+				isDone = true
+			}
+			break
+		}
+		wg.Done()
+
+	}, ants.WithPreAlloc(true))
+	defer p.Release()
+
+	for _, target := range targets {
+		wg.Add(1)
+		_ = p.Invoke(target)
+	}
+	wg.Wait()
+
+}
+
+// GetClouds prepare clouds object in config file
+func GetClouds(options libs.Options) []CloudRunner {
+	var cloudInfos []CloudRunner
+
+	// parse config from cloud/provider.yaml file
+	providerConfigs, err := provider.ParseProvider(options.CloudConfigFile)
+	if err != nil {
+		return cloudInfos
+	}
+
+	options.Cloud.BuildRepo = providerConfigs.Builder.BuildRepo
+	options.Cloud.SecretKey = utils.NormalizePath(providerConfigs.Builder.SecretKey)
+	options.Cloud.PublicKey = utils.NormalizePath(providerConfigs.Builder.PublicKey)
+
+	// we only get provider info from config file but replace token with --token
+	if options.Cloud.IgnoreConfigFile || options.Cloud.Token != "" {
+		var providerConfig provider.ConfigProvider
+		if len(providerConfigs.Clouds) > 0 {
+			utils.InforF("Ignore config file from: %v", options.CloudConfigFile)
+			providerConfig = providerConfigs.Clouds[0]
+		}
+		providerConfig.Token = options.Cloud.Token
+		cloudInfo := SetupProvider(options, providerConfig)
+		cloudInfos = append(cloudInfos, cloudInfo)
+		return cloudInfos
+	}
+
+	for _, providerConfig := range providerConfigs.Clouds {
+		cloudInfo := SetupProvider(options, providerConfig)
+		if len(cloudInfo.Provider.Token) > 6 {
+			cloudInfo.Provider.RedactedToken = cloudInfo.Provider.Token[:5] + "***" + cloudInfo.Provider.Token[len(cloudInfo.Provider.Token)-5:]
+		}
+		cloudInfos = append(cloudInfos, cloudInfo)
+	}
+
+	utils.InforF("Number of cloud providers ready in queue: %v", color.HiMagentaString("%v", len(cloudInfos)))
+	return cloudInfos
+}
+
+// SetupProvider setup new provider
+func SetupProvider(opt libs.Options, providerConfig provider.ConfigProvider) CloudRunner {
+	var cloudRunner CloudRunner
+	cloudRunner.Opt = opt
+	cloudRunner.Prepare()
+
+	providerCloud, err := provider.InitProviderWithConfig(opt, providerConfig)
+	if err != nil {
+		return cloudRunner
+	}
+	cloudRunner.Provider = providerCloud
+
+	cloudRunner.SSHUser = cloudRunner.Provider.SSHUser
+	if cloudRunner.SSHUser == "" {
+		cloudRunner.SSHUser = "root"
+	}
+
+	if cloudRunner.SSHUser != "root" {
+		cloudRunner.BasePath = fmt.Sprintf("/home/%s", cloudRunner.SSHUser)
+	}
+
+	if opt.Cloud.IgnoreSetup {
+		return cloudRunner
+	}
+
+	// check if snapshot is okay or not
+	if !cloudRunner.Provider.SnapshotFound || opt.Cloud.ReBuildBaseImage {
+		err = cloudRunner.Provider.BuildImage()
+		if err != nil {
+			utils.ErrorF("error build snapshot at %v", cloudRunner.Provider.ProviderConfig.BuildFile)
+			return cloudRunner
+		}
+	}
+
+	return cloudRunner
+}
+
+// Prepare some variables
+func (c *CloudRunner) Prepare() {
+	c.SshPrivateKey = c.Opt.Cloud.SecretKey
+	c.SshPublicKey = c.Opt.Cloud.PublicKey
+	c.SSHUser = "root"
+	c.BasePath = "/root"
+
+	// make sure the permission of private key is right
+	os.Chmod(utils.NormalizePath(c.SshPrivateKey), 0600)
+
+	// parse blank target to get env
+	c.Target = core.ParseInput("example.com", c.Opt)
+}

distribute/command.go

@@ -0,0 +1,175 @@
+package distribute
+
+import (
+	"strings"
+	"time"
+
+	"github.com/j3ssie/osmedeus/core"
+	"github.com/j3ssie/osmedeus/utils"
+)
+
+func (c *CloudRunner) CloudMoreParams() {
+	utils.DebugF("Parsed more params")
+
+	// take from -m flag
+	if c.Opt.Cloud.Module != "" {
+		module := core.DirectSelectModule(c.Opt, c.Opt.Cloud.Module)
+		parsedModule, err := core.ParseModules(module)
+		if err != nil || parsedModule.Name == "" {
+			return
+		}
+
+		for _, param := range parsedModule.Params {
+			for k, v := range param {
+				v = core.ResolveData(v, c.Target)
+				if strings.HasPrefix(v, "~/") {
+					v = utils.NormalizePath(v)
+				}
+				c.Target[k] = v
+			}
+		}
+
+		return
+	}
+
+	flows := core.SelectFlow(c.Opt.Cloud.Flow, c.Opt)
+	if len(flows) == 0 {
+		return
+	}
+
+	parsedFlow, err := core.ParseFlow(flows[0])
+	if err != nil {
+		return
+	}
+
+	for _, param := range parsedFlow.Params {
+		for k, v := range param {
+			v = core.ResolveData(v, c.Target)
+			if strings.HasPrefix(v, "~/") {
+				v = utils.NormalizePath(v)
+			}
+			c.Target[k] = v
+		}
+	}
+
+}
+
+func (c *CloudRunner) CreateUIReport() {
+	if c.Opt.NoDB {
+		return
+	}
+
+	// take from -m flag
+	if c.Opt.Cloud.Module != "" {
+		module := core.DirectSelectModule(c.Opt, c.Opt.Cloud.Module)
+		parsedModule, err := core.ParseModules(module)
+		if err != nil || parsedModule.Name == "" {
+			return
+		}
+
+		// create record on UI
+		c.Opt.Module = core.ResolveReports(parsedModule, c.Target)
+		// @TODO: add the report file to the runtime JSON file
+		// database.DBNewReports(parsedModule, &c.Runner.TargetObj)
+		return
+	}
+
+	flows := core.SelectFlow(c.Opt.Cloud.Flow, c.Opt)
+	if len(flows) == 0 {
+		return
+	}
+
+	parsedFlow, err := core.ParseFlow(flows[0])
+	if err != nil {
+		return
+	}
+
+	for _, rawModules := range parsedFlow.Routines {
+		// select module depend on it's flow
+		if rawModules.FlowFolder != "" {
+			parsedFlow.Type = rawModules.FlowFolder
+		} else {
+			parsedFlow.Type = parsedFlow.DefaultType
+		}
+
+		modules := core.SelectModules(rawModules.Modules, c.Opt)
+
+		for _, module := range modules {
+			parsedModule, err := core.ParseModules(module)
+			if err != nil || parsedModule.Desc == "" {
+				continue
+			}
+			// create record on UI
+			c.Opt.Module = core.ResolveReports(parsedModule, c.Target)
+			// @TODO: add the report runtime JSON file
+			// database.DBNewReports(c.Opt.Module, &c.Runner.TargetObj)
+		}
+	}
+}
+
+func (c *CloudRunner) RetryCommandWithExpectString(cmd string, expectString string, timeoutRaw ...string) string {
+	timeout := "300s"
+	if len(timeoutRaw) > 0 {
+		timeout = timeoutRaw[0]
+	}
+	var out string
+
+	utils.DebugF("Retry command: %s", cmd)
+	for i := 0; i < c.Opt.Cloud.Retry; i++ {
+		if timeout == "000" {
+			out, _ = utils.RunOSCommand(cmd)
+		} else {
+			out = utils.RunCmdWithOutput(cmd, timeout)
+		}
+
+		if !strings.Contains(out, expectString) {
+			utils.DebugF(out)
+			time.Sleep(time.Duration(60*(i+1)) * time.Second)
+			continue
+		}
+		return out
+	}
+	return out
+}
+
+func (c *CloudRunner) RetryCommandWithExcludeString(cmd string, excludeString string, timeoutRaw ...string) string {
+	var out string
+	timeout := "300s"
+	if len(timeoutRaw) > 0 {
+		timeout = timeoutRaw[0]
+	}
+
+	utils.DebugF("Retry command: %s", cmd)
+	for i := 0; i < c.Opt.Cloud.Retry; i++ {
+		if timeout == "000" {
+			out, _ = utils.RunOSCommand(cmd)
+		} else {
+			out = utils.RunCmdWithOutput(cmd, timeout)
+		}
+
+		if strings.Contains(out, excludeString) {
+			utils.DebugF(out)
+			time.Sleep(time.Duration(60*(i+1)) * time.Second)
+			continue
+		}
+		return out
+	}
+	return out
+}
+
+func (c *CloudRunner) RetryCommand(cmd string, timeoutRaw ...string) {
+	timeout := "300s"
+	if len(timeoutRaw) > 0 {
+		timeout = timeoutRaw[0]
+	}
+	utils.DebugF("Retry command: %s", cmd)
+	for i := 0; i < c.Opt.Cloud.Retry; i++ {
+		out, err := utils.RunCommandWithErr(cmd, timeout)
+		if err != nil {
+			utils.DebugF(out)
+			time.Sleep(time.Duration(60*(i+1)) * time.Second)
+			continue
+		}
+		break
+	}
+}

distribute/create.go

@@ -0,0 +1,153 @@
+package distribute
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/fatih/color"
+	"github.com/j3ssie/osmedeus/core"
+	"github.com/j3ssie/osmedeus/libs"
+	"github.com/j3ssie/osmedeus/utils"
+	"github.com/jinzhu/copier"
+	jsoniter "github.com/json-iterator/go"
+)
+
+func (c *CloudRunner) CreateInstance(target string) error {
+	c.Opt.Cloud.Input = target
+	c.Target = core.ParseInput(target, c.Opt)
+
+	if c.Opt.EnableFormatInput {
+		c.Opt.Cloud.Input = c.Target["Target"]
+	}
+	if c.Opt.Cloud.Workspace == "" {
+		c.Opt.Cloud.Workspace = utils.CleanPath(c.Target["Target"])
+	}
+
+	if c.Opt.Cloud.Workspace != "" {
+		c.Target["Workspace"] = c.Opt.Cloud.Workspace
+		// disable changing workspace name in huntersuite to keep track the scanID
+		if c.Opt.Cloud.EnableChunk {
+			if strings.Contains(target, "-chunk-") {
+				index := strings.Split(target, "-chunk-")[1]
+				c.Target["WorkspaceChunk"] = path.Base(c.Opt.Cloud.Workspace) + "-" + index
+				utils.DebugF("Changing workspace name: %v", c.Target["WorkspaceChunk"])
+			}
+		}
+		utils.DebugF(`c.Target["Workspace"] -- %v`, c.Target["Workspace"])
+	}
+
+	// run-flow-example.com
+	InstancePrefix := fmt.Sprintf("run")
+	if c.Opt.Cloud.Flow != "" {
+		InstancePrefix = fmt.Sprintf("run-%s", utils.CleanPath(c.Opt.Cloud.Flow))
+	}
+	if c.Opt.Cloud.Module != "" {
+		InstancePrefix = fmt.Sprintf("run-%s", utils.CleanPath(c.Opt.Cloud.Module))
+	}
+
+	// run-flow-example.com-1
+	c.InstanceName = fmt.Sprintf("%s-%s", InstancePrefix, strings.TrimSpace(c.Target["Workspace"]))
+	if c.Opt.Cloud.EnableChunk {
+		c.InstanceName = fmt.Sprintf("%s-%s", InstancePrefix, strings.TrimSpace(c.Target["WorkspaceChunk"]))
+	}
+
+	// make sure the droplet name is unique
+	c.InstanceName = c.InstanceName + "-" + utils.RandomString(4)
+	// clean up the instance name first
+	if strings.Contains(c.InstanceName, "_") {
+		c.InstanceName = strings.ReplaceAll(c.InstanceName, "_", "-")
+	}
+
+	// force changing instance name from cli
+	if c.Opt.Cloud.InstanceName != "" {
+		c.InstanceName = c.Opt.Cloud.InstanceName
+	}
+
+	// quick check for instance name
+	switch c.Provider.ProviderName {
+	case "ln", "line", "linode":
+		if len(c.InstanceName) > 32 {
+			c.InstanceName = strings.Trim(strings.Trim(strings.Trim(c.InstanceName[:20], "-"), "."), "_") + "-" + utils.RandomString(9)
+		}
+	}
+
+	/* Really start to run command to create instance here */
+	err := c.Provider.CreateInstance(c.InstanceName)
+
+	// fix the naming in linode might need another check for length
+	if err != nil {
+		// check if account reach limit first
+		if strings.Contains(err.Error(), "Account Limit reached") {
+			utils.ErrorF("Account %v reach limit instance", c.Provider.RedactedToken)
+			return fmt.Errorf("error creating instance")
+		}
+
+		if strings.Contains(err.Error(), "valid hostname characters are allowed") || strings.Contains(err.Error(), "[400] [label]") {
+			c.InstanceName = fmt.Sprintf("runr-%s-%s", utils.GetTS(), utils.RandomString(8))
+		}
+		err = c.Provider.CreateInstance(c.InstanceName)
+		if err != nil {
+			return fmt.Errorf("error creating instance")
+		}
+	}
+
+	if err != nil {
+		return fmt.Errorf("error creating instance: %v", c.InstanceName)
+	}
+
+	c.PublicIP = c.Provider.CreatedInstance.IPAddress
+
+	c.InstanceID = c.Provider.CreatedInstance.InstanceID
+	c.DestInstance = fmt.Sprintf("%s@%s", c.SSHUser, c.PublicIP)
+	utils.InforF("Instance created: %s -- %s -- %v", color.HiCyanString(c.InstanceName), color.HiCyanString(c.InstanceID), color.HiCyanString(c.PublicIP))
+	c.Target["CIP"] = c.PublicIP
+	c.Target["RemoteIP"] = c.PublicIP
+
+	// create the JSON file for the instance
+	c.WriteInstanceConfig()
+	return nil
+
+}
+
+func (c *CloudRunner) DeleteInstanceConfig() {
+	utils.DebugF("Deleting instance config %v", color.HiCyanString(c.InstanceFile))
+	os.Remove(c.InstanceFile)
+}
+
+func (c *CloudRunner) WriteInstanceConfig() error {
+	instanceFile := c.Opt.Env.InstancesFolder + "/" + c.InstanceName + "-" + c.PublicIP + ".json"
+	c.InstanceFile = instanceFile
+	utils.DebugF("Writing instance config to %v", color.HiCyanString(instanceFile))
+
+	var instanceObj CloudRunner
+	copier.Copy(&instanceObj, &c)
+	instanceObj.Runner = core.Runner{}
+	instanceObj.Opt = libs.Options{}
+
+	if data, err := jsoniter.MarshalToString(instanceObj); err == nil {
+		utils.WriteToFile(instanceFile, data)
+	}
+
+	return nil
+}
+
+func (c *CloudRunner) ErrorWriteInstanceConfig() error {
+	instanceFile := c.Opt.Env.InstancesFolder + "/" + c.InstanceName + "-" + c.PublicIP + ".json"
+	c.InstanceFile = instanceFile
+	utils.DebugF("Writing instance config to %v", color.HiCyanString(instanceFile))
+
+	var instanceObj CloudRunner
+	copier.Copy(&instanceObj, &c)
+
+	instanceObj.IsError = true
+	instanceObj.Runner = core.Runner{}
+	instanceObj.Opt = libs.Options{}
+
+	if data, err := jsoniter.MarshalToString(instanceObj); err == nil {
+		utils.WriteToFile(instanceFile, data)
+	}
+
+	return nil
+}

distribute/db_cloud.go

@@ -0,0 +1,35 @@
+package distribute
+
+import (
+	"path"
+	"time"
+
+	"github.com/fatih/color"
+	"github.com/j3ssie/osmedeus/database"
+	"github.com/j3ssie/osmedeus/utils"
+	jsoniter "github.com/json-iterator/go"
+)
+
+func (c *CloudRunner) DBNewTarget() {
+	targetFolder := path.Join(c.Opt.Env.WorkspacesFolder, c.Target["Workspace"])
+	utils.MakeDir(targetFolder)
+
+	c.Runner.ScanObj = database.Scan{
+		IsCloud:   true,
+		InputName: c.Target["Workspace"],
+		TaskType:  "distributed",
+	}
+
+	c.Runner.ScanObj.CreatedAt = time.Now()
+	c.DBRuntimeUpdate()
+}
+
+func (c *CloudRunner) DBRuntimeUpdate() {
+	runtimeFile := path.Join(c.Opt.Env.WorkspacesFolder, c.Target["Workspace"], "runtime")
+	c.Runner.ScanObj.UpdatedAt = time.Now()
+
+	if runtimeData, err := jsoniter.MarshalToString(c.Runner.ScanObj); err == nil {
+		utils.InforF("Updating runtime file: %s", color.HiCyanString(runtimeFile))
+		utils.WriteToFile(runtimeFile, runtimeData)
+	}
+}

distribute/health.go

@@ -0,0 +1,159 @@
+package distribute
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/fatih/color"
+	"github.com/j3ssie/osmedeus/libs"
+	"github.com/j3ssie/osmedeus/utils"
+	"github.com/jinzhu/copier"
+	jsoniter "github.com/json-iterator/go"
+	"github.com/panjf2000/ants"
+)
+
+func GetAllInstances(opt libs.Options) (instances []CloudRunner) {
+	utils.InforF("Getting all instances from %s", opt.Env.InstancesFolder)
+
+	// Read all entries in the directory
+	entries, err := os.ReadDir(opt.Env.InstancesFolder)
+	if err != nil {
+		utils.ErrorF("error reading instances folder: %v", err)
+		return instances
+	}
+
+	for _, entry := range entries {
+		instanceFile := filepath.Join(opt.Env.InstancesFolder, entry.Name())
+
+		if !utils.FileExists(instanceFile) {
+			continue
+		}
+
+		instance := CloudRunner{}
+		runtimeContent := utils.GetFileContent(instanceFile)
+		err := jsoniter.UnmarshalFromString(runtimeContent, &instance)
+		if err != nil {
+			utils.ErrorF("error marshal data: %v", err)
+		}
+		instances = append(instances, instance)
+	}
+
+	return instances
+}
+
+func CheckingCloudInstance(opt libs.Options) {
+	if opt.Cloud.Retry == 0 {
+		opt.Cloud.Retry = 8
+	}
+
+	// select all cloud instance
+	instances := GetAllInstances(opt)
+	if len(instances) == 0 {
+		utils.WarnF("no active cloud instance running")
+		return
+	}
+
+	var wg sync.WaitGroup
+	p, _ := ants.NewPoolWithFunc(opt.Concurrency*5, func(i interface{}) {
+		// really start to scan
+		instance := i.(CloudRunner)
+
+		var options libs.Options
+		copier.Copy(&options, &opt)
+		instance.Opt = opt
+		instance.Provider.IsBackgroundCheck = true
+		instance.Provider.InitClient()
+		instance.Prepare()
+		instance.HealthCheck()
+		wg.Done()
+	}, ants.WithPreAlloc(true))
+	defer p.Release()
+
+	utils.InforF("Evaluating the status of %v cloud instances", color.HiMagentaString("%v", len(instances)))
+	for _, instance := range instances {
+		wg.Add(1)
+		_ = p.Invoke(instance)
+	}
+	wg.Wait()
+
+}
+
+func (c *CloudRunner) HealthCheck() bool {
+	utils.InforF("[%v] Checking the instance: %v -- %v", color.HiYellowString(c.Provider.ProviderName), color.HiCyanString(c.PublicIP), color.HiCyanString(path.Base(c.InstanceName)))
+	c.Opt.Cloud.Input = c.Input
+
+	counter := 0
+	for i := 0; i < c.Opt.Cloud.Retry; i++ {
+		// sync in result first
+		c.SyncResult()
+		if !c.IsRunning() || c.IsPanic() {
+			counter += 1
+			utils.DebugF("retry[%v]: An error has been detected at the cloud instance: %v -- %v", counter, c.Provider.ProviderName, c.PublicIP)
+		}
+
+		if counter == c.Opt.Cloud.Retry-1 {
+			utils.ErrorF("An error has been detected at the cloud instance: %v -- %v", c.Provider.ProviderName, c.PublicIP)
+			err := c.Provider.DeleteInstance(c.InstanceID)
+			if err == nil {
+				utils.InforF("Instance deleted %s -- %s", color.HiYellowString(c.PublicIP), color.HiYellowString(c.InstanceID))
+				instanceFile := c.Opt.Env.InstancesFolder + "/" + c.InstanceName + "-" + c.PublicIP + ".json"
+				os.Remove(instanceFile)
+			}
+			return false
+		}
+		time.Sleep(10 * time.Second)
+	}
+
+	utils.InforF("[%v] Instance is still running well: %v -- %v", color.HiYellowString(c.Provider.ProviderName), color.HiCyanString(c.PublicIP), color.HiCyanString(path.Base(c.InstanceName)))
+	return true
+}
+
+// IsRunning checking if cloud instance is running or not
+func (c *CloudRunner) IsRunning() bool {
+	utils.DebugF("Checking running process at: %v", c.PublicIP)
+	cmd := fmt.Sprintf("%s utils ps --json", libs.BINARY)
+
+	// ignore checking process if you're running custom command '--no-ps'
+	if c.Opt.Cloud.IgnoreProcess {
+		return true
+	}
+
+	out, err := c.SSHExec(cmd)
+	if err == nil && strings.Contains(out, "pid") {
+		return true
+	}
+
+	// retry checking process
+	for i := 0; i < c.Opt.Cloud.Retry; i += 2 {
+		out, err := c.SSHExec(cmd)
+		if err == nil && strings.Contains(out, "pid") {
+			return true
+		}
+	}
+
+	utils.DebugF(out)
+	utils.ErrorF("no process running at %v", c.PublicIP)
+	return false
+}
+
+// IsPanic checking if cloud instance has any panic error
+func (c *CloudRunner) IsPanic() bool {
+	utils.DebugF("Checking panic error at: %v", c.PublicIP)
+	cmd := fmt.Sprintf("%s utils tmux logs -A -l 30", libs.BINARY)
+	out, err := c.SSHExec(cmd)
+
+	if err == nil {
+		if strings.Contains(out, "out of memory") || strings.Contains(out, "runtime.(*") || strings.Contains(out, "[panic]") {
+			utils.DebugF(out)
+			utils.ErrorF("Fatal panic detected at: %s", c.PublicIP)
+			return true
+		}
+	}
+
+	return false
+}

distribute/mics.go

@@ -0,0 +1,115 @@
+package distribute
+
+import (
+	"fmt"
+	"path"
+	"strings"
+
+	"github.com/j3ssie/osmedeus/libs"
+	"github.com/j3ssie/osmedeus/utils"
+	"github.com/spf13/cast"
+)
+
+// CommandBuilder build core command from API
+func CommandBuilder(options libs.Options) string {
+	binary := libs.BINARY + " --from-remote"
+
+	if options.Debug {
+		binary += " --debug"
+	}
+
+	if options.Update.EnableUpdate {
+		binary += " --update"
+	}
+
+	if options.Threads > 0 {
+		binary += " --threads-hold " + cast.ToString(options.Threads)
+	}
+
+	if options.ScanID != "" {
+		binary += fmt.Sprintf(" --sid %v ", options.ScanID)
+	}
+
+	taskData := options.Cloud
+	var command string
+	var workspace, concurrency, timeout, params, workflow, plugin, extra string
+	if options.Cloud.Extra != "" {
+		extra = " " + options.Cloud.Extra + " "
+	}
+
+	if options.Cloud.Flow != "" {
+		options.Cloud.Flow = utils.NormalizePath(options.Cloud.Flow)
+		workflow = fmt.Sprintf(" -f '%v' ", options.Cloud.Flow)
+	}
+
+	if len(taskData.Params) > 0 {
+		for _, param := range taskData.Params {
+			params += fmt.Sprintf(" -p '%v'", param)
+		}
+	}
+
+	if options.Cloud.Threads > 1 {
+		concurrency = fmt.Sprintf(" -c %d ", options.Cloud.Threads)
+	}
+
+	// get workspace
+	if taskData.Workspace == "" {
+		taskData.Workspace = utils.CleanPath(taskData.Input)
+	}
+	if taskData.Workspace != "" {
+		workspace = fmt.Sprintf(" -w '%v'", strings.TrimSpace(taskData.Workspace))
+	}
+
+	if taskData.Module != "" {
+		taskData.Module = utils.NormalizePath(taskData.Module)
+		plugin = fmt.Sprintf(" -m '%v' ", taskData.Module)
+	}
+
+	// override everything
+	if taskData.RawCommand != "" {
+		return taskData.RawCommand
+	}
+
+	// use target as a file
+	if options.Cloud.TargetAsFile && utils.FileExists(taskData.Input) {
+		taskData.InputsFile = taskData.Input
+	}
+
+	// mean general scan
+	if taskData.Module == "" {
+		command = fmt.Sprintf("%v scan %v -t %v %v%v%v%v%v", binary, workflow, taskData.Input, concurrency, timeout, workspace, params, extra)
+		if options.Cloud.TargetAsFile {
+			command = fmt.Sprintf("%v scan %v -T %v %v%v%v%v%v", binary, workflow, taskData.InputsFile, concurrency, timeout, workspace, params, extra)
+		}
+		command = strings.TrimSpace(command)
+
+		return command
+	}
+
+	command = fmt.Sprintf("%v scan %v -t %v %v%v%v%v%v", binary, plugin, taskData.Input, concurrency, timeout, workspace, params, extra)
+	if options.Cloud.TargetAsFile {
+		command = fmt.Sprintf("%v scan %v -T %v %v%v%v%v%v", binary, plugin, taskData.InputsFile, concurrency, timeout, workspace, params, extra)
+	}
+	command = strings.TrimSpace(command)
+
+	return command
+}
+
+// PrepareTarget change the target file destination
+func PrepareTarget(target string, options libs.Options) string {
+	if options.Cloud.EnableChunk {
+		return target
+	}
+
+	if !utils.FileExists(target) && !utils.FolderExists(target) {
+		utils.DebugF("target is not a file: %s", target)
+		return target
+	}
+
+	utils.MakeDir(options.Cloud.TempTarget)
+	dest := path.Join(options.Cloud.TempTarget, path.Base(target))
+
+	utils.Copy(target, dest)
+	utils.InforF("Change target %s --> %s", target, dest)
+	return dest
+}

distribute/routine.go

@@ -0,0 +1,94 @@
+package distribute
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/j3ssie/osmedeus/core"
+	"github.com/j3ssie/osmedeus/libs"
+	"github.com/j3ssie/osmedeus/utils"
+)
+
+func (c *CloudRunner) Scan(target string) error {
+	err := c.CreateInstance(target)
+	if err != nil {
+		utils.ErrorF("Error to create instance")
+		return err
+	}
+
+	if c.Opt.Cloud.OnlyCreateDroplet {
+		utils.DebugF("Only create instance, skip the scan")
+		return nil
+	}
+
+	// parse some parameters first
+	c.PrepareInput()
+
+	// pre run before starting the scan
+	c.PreRunLocal()
+
+	if c.Opt.Cloud.EnableSyncWorkflow {
+		err = c.CopyWorkflow()
+		if err != nil {
+			utils.ErrorF("Error to copy workflow to instance")
+			return err
+		}
+	}
+
+	// copy target to droplet first
+	err = c.CopyTarget()
+	if err != nil {
+		utils.ErrorF("Error to copy input to instance")
+		return err
+	}
+
+	// pre run commands
+	c.PreRunRemote()
+
+	err = c.StartScan()
+	if err != nil {
+		utils.ErrorF("Error to run command on instance")
+		return err
+	}
+
+	// check if done file created in instance or not
+	c.CheckingDone()
+
+	if !c.Opt.Cloud.DisableLocalSync {
+		err = c.SyncResult()
+		// sync result to one more time if it's not done yet
+		if c.Opt.Cloud.NoDelete {
+			time.Sleep(100 * time.Second)
+			err = c.SyncResult()
+		}
+
+		if err != nil {
+			utils.ErrorF("Error to sync result to instance")
+			return err
+		}
+	}
+
+	// post run after scan done
+	c.PostRunLocal()
+
+	if c.Opt.Cloud.CopyWorkspaceToGit {
+		utils.InforF("Coping workspace to git storages")
+		baseCmd := fmt.Sprintf("%s scan --nn -f sync -t {{.Workspace}}", libs.BINARY)
+		cmd := core.ResolveData(baseCmd, c.Target)
+		utils.RunCommandWithErr(cmd)
+	}
+
+	if c.Opt.Cloud.NoDelete {
+		utils.DebugF("Skipping delete instance")
+		return nil
+	}
+
+	err = c.Provider.DeleteInstance(c.InstanceID)
+	if err != nil {
+		utils.ErrorF("Error to delete instance")
+		return err
+	}
+	c.DeleteInstanceConfig()
+
+	return nil
+}

distribute/scan.go

@@ -0,0 +1,267 @@
+package distribute
+
+import (
+	"fmt"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/fatih/color"
+	"github.com/j3ssie/osmedeus/core"
+	"github.com/j3ssie/osmedeus/libs"
+	"github.com/j3ssie/osmedeus/provider"
+	"github.com/j3ssie/osmedeus/utils"
+)
+
+func (c *CloudRunner) PrepareInput() {
+	c.Opt.Scan.ROptions = c.Target
+	c.Opt.Scan.Flow = "cloud-distributed"
+	//database.NewScan(c.Opt, "cli")
+	c.Input = c.Target["Target"]
+	runner, err := core.InitRunner(c.Input, c.Opt)
+	if err == nil {
+		c.Runner = runner
+		c.Runner.RunnerType = "cloud"
+	}
+
+	// for creating local DB record
+	if c.Opt.Cloud.Flow != "" {
+		c.TaskName = c.Opt.Cloud.Flow
+		c.TaskType = "flow"
+	} else {
+		c.TaskName = c.Opt.Cloud.Flow
+		c.TaskType = "module"
+	}
+	c.CloudMoreParams()
+	// more params from -p flag
+	if len(c.Opt.Cloud.Params) > 0 {
+		params := core.ParseParams(c.Opt.Cloud.Params)
+		if len(params) > 0 {
+			for k, v := range params {
+				v = core.ResolveData(v, c.Target)
+				c.Target[k] = v
+			}
+		}
+	}
+
+}
+
+func (c *CloudRunner) StartScan() error {
+	c.DBNewTarget()
+	// c.DBNewScanLocal()
+	// c.DBNewCloudInstance()
+
+	err := c.RunScan()
+	if err != nil {
+		return fmt.Errorf("error to start the scan")
+	}
+
+	// utils.DebugF("Create UI report for %s: %s", c.DestInstance, color.HiCyanString(c.Opt.Cloud.RawCommand))
+	// c.Runner.DBDoneScan()
+	return nil
+}
+
+func (c *CloudRunner) RunScan() error {
+	// -f mean run in a background
+	// ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i <private_key> root@IP  -f <command>
+	// -t mean run and wait
+	// ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i <private_key> root@IP  -t <command>
+	if c.Opt.Cloud.RawCommand != "" {
+		c.Opt.Cloud.RawCommand = core.ResolveData(c.Opt.Cloud.RawCommand, c.Target)
+	} else {
+		c.Opt.Cloud.RawCommand = CommandBuilder(c.Opt)
+	}
+	c.Opt.Cloud.RawCommand = core.ResolveData(c.Opt.Cloud.RawCommand, c.Target)
+	c.RawCommand = c.Opt.Cloud.RawCommand
+
+	// init tmux session
+	out, err := c.SSHExec("tmux new-session -d -t main")
+
+	// boot the instance again if it still didn't up
+	if err != nil && strings.Contains(err.Error(), "time out") {
+		time.Sleep(60 * time.Second)
+		c.Provider.Action(provider.BootInstance, c.InstanceID)
+		out, err = c.SSHExec("tmux new-session -d -t main")
+	}
+
+	// really run main command
+	tcmd := fmt.Sprintf(`"%s"`, c.Opt.Cloud.RawCommand)
+	_, err = c.SSHExec(fmt.Sprintf(`tmux send-keys %s ENTER`, tcmd))
+
+	// still error then it must be something wrong
+	if err != nil {
+		utils.ErrorF("An error occurred with %v", color.HiYellowString(c.DestInstance))
+		utils.ErrorF("error log: %v", out)
+		return fmt.Errorf("error running command on %v", color.HiYellowString(c.DestInstance))
+	}
+	utils.InforF("Start to run the scan %v with command %v", color.HiYellowString(c.DestInstance), color.HiCyanString(c.Opt.Cloud.RawCommand))
+
+	// wait a bit for process really start
+	time.Sleep(60 * time.Second)
+	if !c.IsRunning() {
+		return fmt.Errorf("Failed to initiate the scan on %v", color.HiYellowString(c.DestInstance))
+	}
+
+	c.WriteInstanceConfig()
+	return nil
+}
+
+func (c *CloudRunner) CheckingDone() error {
+	if c.Opt.Cloud.NoDelete {
+		time.Sleep(60 * time.Second)
+		return nil
+	}
+	utils.InforF("Checking scan process at: %s", color.HiBlueString(c.PublicIP))
+
+	// dest := fmt.Sprintf("%s/.%s/workspaces/%s/done", c.BasePath, libs.BINARY, c.Target["Workspace"])
+	// @NOTE: this is new workspaces folder
+	dest := fmt.Sprintf("%s/workspaces-%s/%s/done", c.BasePath, libs.BINARY, c.Target["Workspace"])
+
+	cmd := fmt.Sprintf("file %s", dest)
+	out, _ := c.SSHExec(cmd)
+
+	if strings.Contains(out, "ASCII text") || strings.Contains(out, "JSON data") {
+		return nil
+	}
+
+	waitTime := utils.CalcTimeout(c.Opt.Cloud.ClearTime)
+	counter := 1
+	for {
+		time.Sleep(time.Duration(waitTime) * time.Second)
+		out, _ = c.SSHExec(cmd)
+		if strings.Contains(out, "ASCII text") || strings.Contains(out, "JSON data") {
+			utils.InforF("The scan is done at: %s", color.HiBlueString(c.PublicIP))
+			return nil
+		}
+
+		if !c.IsRunning() {
+			return fmt.Errorf("no process running at %v", c.PublicIP)
+		}
+
+		// check if we have panic or not
+		if c.IsPanic() {
+			return fmt.Errorf("panic detected at %v", c.PublicIP)
+		}
+
+		if counter%50 == 0 {
+			c.SyncResult()
+		}
+		counter++
+	}
+}
+
+// below code is  experimental part
+
+func (c *CloudRunner) SyncResult() error {
+	target := c.Opt.Cloud.Input
+	if !c.Provider.IsBackgroundCheck {
+		utils.InforF("Sync back the data of taget %v from %v", color.HiCyanString(target), color.HiYellowString(c.DestInstance))
+	}
+
+	if c.Opt.Cloud.LocalSyncFolder == "" {
+		c.Opt.Cloud.LocalSyncFolder = fmt.Sprintf("%s/workspaces-%s/", c.BasePath, libs.BINARY)
+	}
+
+	// on vps machine
+	src := c.Opt.Cloud.LocalSyncFolder
+
+	// on local
+	dest := path.Join(c.Opt.Env.WorkspacesFolder, c.Opt.Cloud.BaseWorkspace)
+	utils.MakeDir(dest)
+
+	cmd := fmt.Sprintf("rsync -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s' -avzr --progress %s:%s %s", c.SshPrivateKey, c.DestInstance, src, dest)
+
+	c.RetryCommandWithExpectString(cmd, `bytes/sec`)
+	if !utils.FolderExists(dest) {
+		utils.ErrorF("error sync result back from: %v to %v", c.DestInstance, dest)
+	}
+
+	return nil
+}
+
+func (c *CloudRunner) CopyTarget() error {
+	target := c.Opt.Cloud.Input
+	utils.DebugF("Sync input of %s to %s", target, c.DestInstance)
+
+	dest := c.Target["Target"]
+	if !utils.FileExists(dest) && !utils.FolderExists(dest) {
+		utils.DebugF("target is not a file: %s", dest)
+		return nil
+	}
+
+	if c.Opt.Cloud.EnableChunk {
+		dest = c.Opt.Cloud.ChunkInputs
+	}
+
+	c.SSHExec(fmt.Sprintf("mkdir -p %s", path.Dir(dest)))
+	cmd := fmt.Sprintf("rsync -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s' -avzr --progress %s %s:%s", c.SshPrivateKey, dest, c.DestInstance, dest)
+	c.RetryCommandWithExpectString(cmd, `bytes/sec`)
+	return nil
+}
+
+func (c *CloudRunner) CopyWorkflow() error {
+	utils.DebugF("Sync workflow of %s to %s", c.Opt.Env.WorkFlowsFolder, c.DestInstance)
+	destWorkflow := fmt.Sprintf("%v/osmedeus-base/", c.BasePath)
+	if c.Opt.Cloud.RemoteWorkflowFolder != "" {
+		destWorkflow = c.Opt.Cloud.RemoteWorkflowFolder
+	}
+
+	// c.SSHExec(fmt.Sprintf("rm -rf %s && mkdir -p %s", destWorkflow, destWorkflow))
+	cmd := fmt.Sprintf("rsync -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s' -avzr --progress %s %s:%s", c.SshPrivateKey, c.Opt.Env.WorkFlowsFolder, c.DestInstance, destWorkflow)
+	c.RetryCommandWithExpectString(cmd, `bytes/sec`)
+	return nil
+}
+
+func (c *CloudRunner) PreRunRemote() error {
+	if len(c.Opt.Cloud.RemotePreRun) <= 0 {
+		return nil
+	}
+	utils.InforF("Run remote command on: %s", c.PublicIP)
+
+	// really start to run pre commands
+	for _, rcmd := range c.Opt.Cloud.RemotePreRun {
+		cmd := core.ResolveData(rcmd, c.Target)
+		utils.InforF("Run pre command on %s: %s", c.PublicIP, cmd)
+		c.SSHExec(cmd)
+	}
+	return nil
+}
+
+func (c *CloudRunner) PreRunLocal() error {
+	if len(c.Opt.Cloud.LocalPreRun) <= 0 {
+		return nil
+	}
+	c.Opt.Scan.ROptions = c.Target
+	utils.InforF("Start %v", color.HiCyanString("PreRunLocal"))
+
+	// really start to run pre commands
+	for _, script := range c.Opt.Cloud.LocalPreRun {
+		script = core.ResolveData(script, c.Target)
+		c.Runner.RunScript(script)
+	}
+	return nil
+}
+
+func (c *CloudRunner) PostRunLocal() error {
+	c.Opt.Scan.ROptions = c.Target
+
+	if len(c.Opt.Cloud.LocalSteps) > 0 {
+		// for running local steps
+		utils.DebugF("Running local steps")
+		for _, step := range c.Opt.Cloud.LocalSteps {
+			c.Runner.RunStep(step)
+		}
+	}
+
+	if len(c.Opt.Cloud.LocalPostRun) <= 0 {
+		return nil
+	}
+
+	utils.InforF("Start %v", color.HiCyanString("PostRunLocal"))
+	// really start to run pre commands
+	for _, script := range c.Opt.Cloud.LocalPostRun {
+		script = core.ResolveData(script, c.Target)
+		c.Runner.RunScript(script)
+	}
+	return nil
+}

distribute/ssh.go

@@ -0,0 +1,430 @@
+package distribute
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/fatih/color"
+	"github.com/j3ssie/osmedeus/utils"
+
+	"golang.org/x/crypto/ssh"
+)
+
+func (c *CloudRunner) SSHExec(command string) (string, error) {
+	client, err := c.InitSSHClient()
+	if err != nil {
+		utils.ErrorF("Failed to initialize SSH connection to %v", color.HiYellowString(c.PublicIP))
+		return "", err
+	}
+	defer client.Close()
+
+	utils.DebugF("Running command on %s: %s", color.HiBlueString(c.PublicIP), color.HiGreenString(command))
+	out, err := client.Cmd(command).Output()
+	if err != nil {
+		utils.ErrorF("err run command: %v", err)
+		return "", err
+	}
+
+	return string(out), nil
+}
+
+func (c *CloudRunner) InitSSHClient() (*Client, error) {
+	host := fmt.Sprintf("%s:%s", c.PublicIP, "22")
+	dest := color.HiCyanString("%s@%s", c.SSHUser, host)
+	utils.DebugF("Connecting to %v with key %v", dest, c.SshPrivateKey)
+	client, err := DialWithKeyString(host, c.SSHUser, c.Opt.Cloud.SecretKeyContent)
+	if err != nil {
+		utils.ErrorF("err connect to %v -- %v", host, err)
+		for i := 0; i < c.Opt.Cloud.Retry; i++ {
+			client, err = DialWithKeyString(host, c.SSHUser, c.Opt.Cloud.SecretKeyContent)
+			if err != nil {
+				time.Sleep(time.Duration(30*(i+1)) * time.Second)
+				continue
+			}
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+	}
+	return client, nil
+}
+
+/* Start of SSH Lib */
+// literally copy from this: https://github.com/helloyi/go-sshclient
+
+type remoteScriptType byte
+type remoteShellType byte
+
+const (
+	cmdLine remoteScriptType = iota
+	rawScript
+	scriptFile
+
+	interactiveShell remoteShellType = iota
+	nonInteractiveShell
+)
+
+type Client struct {
+	client *ssh.Client
+}
+
+// DialWithPasswd starts a client connection to the given SSH server with passwd authmethod.
+func DialWithPasswd(addr, user, passwd string) (*Client, error) {
+	config := &ssh.ClientConfig{
+		User: user,
+		Auth: []ssh.AuthMethod{
+			ssh.Password(passwd),
+		},
+		HostKeyCallback: ssh.HostKeyCallback(func(hostname string, remote net.Addr, key ssh.PublicKey) error { return nil }),
+	}
+
+	return Dial("tcp", addr, config)
+}
+
+// DialWithKeyString starts a client connection to the given SSH server with key authmethod.
+func DialWithKeyString(addr, user, keyContent string) (*Client, error) {
+	signer, err := ssh.ParsePrivateKey([]byte(strings.TrimSpace(keyContent)))
+	if err != nil {
+		return nil, err
+	}
+
+	config := &ssh.ClientConfig{
+		User: user,
+		Auth: []ssh.AuthMethod{
+			ssh.PublicKeys(signer),
+		},
+		HostKeyCallback: ssh.HostKeyCallback(func(hostname string, remote net.Addr, key ssh.PublicKey) error { return nil }),
+	}
+
+	return Dial("tcp", addr, config)
+}
+
+// DialWithKey starts a client connection to the given SSH server with key authmethod.
+func DialWithKey(addr, user, keyfile string) (*Client, error) {
+	key, err := os.ReadFile(keyfile)
+	if err != nil {
+		return nil, err
+	}
+
+	signer, err := ssh.ParsePrivateKey(key)
+	if err != nil {
+		return nil, err
+	}
+
+	config := &ssh.ClientConfig{
+		User: user,
+		Auth: []ssh.AuthMethod{
+			ssh.PublicKeys(signer),
+		},
+		HostKeyCallback: ssh.HostKeyCallback(func(hostname string, remote net.Addr, key ssh.PublicKey) error { return nil }),
+	}
+
+	return Dial("tcp", addr, config)
+}
+
+// DialWithKeyWithPassphrase same as DialWithKey but with a passphrase to decrypt the private key
+func DialWithKeyWithPassphrase(addr, user, keyfile string, passphrase string) (*Client, error) {
+	key, err := os.ReadFile(keyfile)
+	if err != nil {
+		return nil, err
+	}
+
+	signer, err := ssh.ParsePrivateKeyWithPassphrase(key, []byte(passphrase))
+	if err != nil {
+		return nil, err
+	}
+
+	config := &ssh.ClientConfig{
+		User: user,
+		Auth: []ssh.AuthMethod{
+			ssh.PublicKeys(signer),
+		},
+		HostKeyCallback: ssh.HostKeyCallback(func(hostname string, remote net.Addr, key ssh.PublicKey) error { return nil }),
+	}
+
+	return Dial("tcp", addr, config)
+}
+
+// Dial starts a client connection to the given SSH server.
+// This is wrap the ssh.Dial
+func Dial(network, addr string, config *ssh.ClientConfig) (*Client, error) {
+	client, err := ssh.Dial(network, addr, config)
+	if err != nil {
+		return nil, err
+	}
+	return &Client{
+		client: client,
+	}, nil
+}
+
+func (c *Client) Close() error {
+	return c.client.Close()
+}
+
+// Cmd create a command on client
+func (c *Client) Cmd(cmd string) *remoteScript {
+	return &remoteScript{
+		_type:  cmdLine,
+		client: c.client,
+		script: bytes.NewBufferString(cmd + "\n"),
+	}
+}
+
+// Script
+func (c *Client) Script(script string) *remoteScript {
+	return &remoteScript{
+		_type:  rawScript,
+		client: c.client,
+		script: bytes.NewBufferString(script + "\n"),
+	}
+}
+
+// ScriptFile
+func (c *Client) ScriptFile(fname string) *remoteScript {
+	return &remoteScript{
+		_type:      scriptFile,
+		client:     c.client,
+		scriptFile: fname,
+	}
+}
+
+type remoteScript struct {
+	client     *ssh.Client
+	_type      remoteScriptType
+	script     *bytes.Buffer
+	scriptFile string
+	err        error
+
+	stdout io.Writer
+	stderr io.Writer
+}
+
+// Run
+func (rs *remoteScript) Run() error {
+	if rs.err != nil {
+		fmt.Println(rs.err)
+		return rs.err
+	}
+
+	if rs._type == cmdLine {
+		return rs.runCmds()
+	} else if rs._type == rawScript {
+		return rs.runScript()
+	} else if rs._type == scriptFile {
+		return rs.runScriptFile()
+	} else {
+		return errors.New("Not supported remoteScript type")
+	}
+}
+
+func (rs *remoteScript) Output() ([]byte, error) {
+	if rs.stdout != nil {
+		return nil, errors.New("Stdout already set")
+	}
+	var out bytes.Buffer
+	rs.stdout = &out
+	err := rs.Run()
+	return out.Bytes(), err
+}
+
+func (rs *remoteScript) SmartOutput() ([]byte, error) {
+	if rs.stdout != nil {
+		return nil, errors.New("Stdout already set")
+	}
+	if rs.stderr != nil {
+		return nil, errors.New("Stderr already set")
+	}
+
+	var (
+		stdout bytes.Buffer
+		stderr bytes.Buffer
+	)
+	rs.stdout = &stdout
+	rs.stderr = &stderr
+	err := rs.Run()
+	if err != nil {
+		return stderr.Bytes(), err
+	}
+	return stdout.Bytes(), err
+}
+
+func (rs *remoteScript) Cmd(cmd string) *remoteScript {
+	_, err := rs.script.WriteString(cmd + "\n")
+	if err != nil {
+		rs.err = err
+	}
+	return rs
+}
+
+func (rs *remoteScript) SetStdio(stdout, stderr io.Writer) *remoteScript {
+	rs.stdout = stdout
+	rs.stderr = stderr
+	return rs
+}
+
+func (rs *remoteScript) runCmd(cmd string) error {
+	session, err := rs.client.NewSession()
+	if err != nil {
+		return err
+	}
+	defer session.Close()
+
+	session.Stdout = rs.stdout
+	session.Stderr = rs.stderr
+
+	if err := session.Run(cmd); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (rs *remoteScript) runCmds() error {
+	for {
+		statment, err := rs.script.ReadString('\n')
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return err
+		}
+
+		if err := rs.runCmd(statment); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (rs *remoteScript) runScript() error {
+	session, err := rs.client.NewSession()
+	if err != nil {
+		return err
+	}
+
+	session.Stdin = rs.script
+	session.Stdout = rs.stdout
+	session.Stderr = rs.stderr
+
+	if err := session.Shell(); err != nil {
+		return err
+	}
+	if err := session.Wait(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (rs *remoteScript) runScriptFile() error {
+	var buffer bytes.Buffer
+	file, err := os.Open(rs.scriptFile)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+	_, err = io.Copy(&buffer, file)
+	if err != nil {
+		return err
+	}
+
+	rs.script = &buffer
+	return rs.runScript()
+}
+
+type remoteShell struct {
+	client         *ssh.Client
+	requestPty     bool
+	terminalConfig *TerminalConfig
+
+	stdin  io.Reader
+	stdout io.Writer
+	stderr io.Writer
+}
+
+type TerminalConfig struct {
+	Term   string
+	Height int
+	Weight int
+	Modes  ssh.TerminalModes
+}
+
+// Terminal create a interactive shell on client.
+func (c *Client) Terminal(config *TerminalConfig) *remoteShell {
+	return &remoteShell{
+		client:         c.client,
+		terminalConfig: config,
+		requestPty:     true,
+	}
+}
+
+// Shell create a noninteractive shell on client.
+func (c *Client) Shell() *remoteShell {
+	return &remoteShell{
+		client:     c.client,
+		requestPty: false,
+	}
+}
+
+func (rs *remoteShell) SetStdio(stdin io.Reader, stdout, stderr io.Writer) *remoteShell {
+	rs.stdin = stdin
+	rs.stdout = stdout
+	rs.stderr = stderr
+	return rs
+}
+
+// Start start a remote shell on client
+func (rs *remoteShell) Start() error {
+	session, err := rs.client.NewSession()
+	if err != nil {
+		return err
+	}
+	defer session.Close()
+
+	if rs.stdin == nil {
+		session.Stdin = os.Stdin
+	} else {
+		session.Stdin = rs.stdin
+	}
+	if rs.stdout == nil {
+		session.Stdout = os.Stdout
+	} else {
+		session.Stdout = rs.stdout
+	}
+	if rs.stderr == nil {
+		session.Stderr = os.Stderr
+	} else {
+		session.Stderr = rs.stderr
+	}
+
+	if rs.requestPty {
+		tc := rs.terminalConfig
+		if tc == nil {
+			tc = &TerminalConfig{
+				Term:   "xterm",
+				Height: 40,
+				Weight: 80,
+			}
+		}
+		if err := session.RequestPty(tc.Term, tc.Height, tc.Weight, tc.Modes); err != nil {
+			return err
+		}
+	}
+
+	if err := session.Shell(); err != nil {
+		return err
+	}
+
+	if err := session.Wait(); err != nil {
+		return err
+	}
+
+	return nil
+}

distribute/wizard.go

@@ -0,0 +1,240 @@
+package distribute
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"strings"
+	"syscall"
+
+	"github.com/Shopify/yaml"
+	"github.com/fatih/color"
+	"github.com/j3ssie/osmedeus/libs"
+	"github.com/j3ssie/osmedeus/provider"
+	"github.com/j3ssie/osmedeus/utils"
+	"github.com/olekukonko/tablewriter"
+	"golang.org/x/term"
+)
+
+func InitCloudSetup(opt libs.Options) {
+	var supportedProvider = []string{"aws", "digitalocean", "linode"}
+	fmt.Println("🔮 Start cloud setup wizard 🔮")
+	fmt.Println("Currently only these providers are supported: ", color.HiYellowString("%v", supportedProvider))
+
+	var configProviders provider.ConfigProviders
+	configProviders.Builder.BuildRepo = opt.Cloud.BuildRepo
+	if configProviders.Builder.BuildRepo == "" {
+		configProviders.Builder.BuildRepo = StringPrompt("🌀 Enter premium install script URL (e.g: https://long-url-here/x/premium.sh)?", "")
+	}
+	configProviders.Builder.PublicKey = opt.Cloud.PublicKey
+	configProviders.Builder.SecretKey = opt.Cloud.SecretKey
+
+	// get the existing one
+	if opt.Cloud.AddNewProvider {
+		fmt.Println("🪄 Add new provider config to the existing config")
+		providerConfigs, err := provider.ParseProvider(opt.CloudConfigFile)
+		if err == nil {
+			configProviders.Clouds = providerConfigs.Clouds
+		}
+	}
+
+	for {
+		configProvider := generateProvider()
+		configProviders.Clouds = append(configProviders.Clouds, configProvider)
+		if stop := StringPrompt("🌀 Do you want to add more provider (y/N)?", "n"); stop != "y" {
+			break
+		}
+	}
+
+	data, err := yaml.Marshal(&configProviders)
+	if err != nil {
+		return
+	}
+
+	//utils.DebugF(string(data))
+	if override := StringPrompt("🧙 Do you want to override the old config at "+color.HiGreenString(opt.CloudConfigFile)+" (Y/n)?", "y"); override != "n" {
+		_, err := utils.WriteToFile(opt.CloudConfigFile, string(data))
+		if err != nil {
+			utils.WarnF("error to write provider config: %v", opt.CloudConfigFile)
+		}
+	}
+
+	if isValidate := StringPrompt("❄️  Do you want to validate your provider config (Y/n)?", "y"); isValidate == "n" {
+		return
+	}
+	fmt.Printf("💡 You also can manually rebuild the snapshot with the command: %s\n", color.HiCyanString("%s provider build --rebuild", libs.BINARY))
+
+	var cloudRunners []CloudRunner
+	for _, configProvider := range configProviders.Clouds {
+		cloudRunner, err := ValidateProvider(opt, configProvider)
+		if err != nil {
+			utils.ErrorF("error validate config: %v -- %v", configProvider.Name, configProvider.RedactedToken)
+			continue
+		}
+		cloudRunners = append(cloudRunners, cloudRunner)
+	}
+
+	content := [][]string{}
+	for _, cloudRunner := range cloudRunners {
+		row := []string{
+			cloudRunner.Provider.ProviderName,
+			cloudRunner.Provider.RedactedToken,
+			cloudRunner.Provider.SSHKeyID,
+			cloudRunner.Provider.SnapshotID,
+		}
+		content = append(content, row)
+	}
+	table := tablewriter.NewWriter(os.Stderr)
+	table.SetAutoFormatHeaders(false)
+	table.SetHeader([]string{"Provider", "Token", "SSH Key ID", "Osmedeus Snapshot ID"})
+	table.SetBorders(tablewriter.Border{Left: true, Top: false, Right: true, Bottom: false})
+	table.SetCenterSeparator("|")
+	table.AppendBulk(content) // Add Bulk Data
+	table.Render()
+
+}
+
+func generateProvider() provider.ConfigProvider {
+	configProvider := provider.ConfigProvider{
+		Token:        "",
+		Provider:     "digitalocean",
+		DefaultImage: "debian-10-x64",
+		Size:         "s-2vcpu-4gb",
+		Region:       "sfo3",
+		Username:     "root",
+		Limit:        0,
+	}
+
+	//#   provider: "digitalocean"
+	//#   name: "do-osmp"
+	//#   default_image: "debian-10-x64"
+	//#   size: "s-2vcpu-4gb"
+	//#   region: "sfo3"
+
+	//#   provider: "linode"
+	//#   name: "linode-osmp"
+	//#   default_image: "linode/debian10"
+	//#   size: "g6-standard-1"
+	//#   region: "us-east"
+
+	configProvider.Provider = StringPrompt("🌀 What is your cloud provider?", configProvider.Provider)
+
+	switch configProvider.Provider {
+	case "do", "digitalocean":
+		configProvider.Provider = "digitalocean"
+		fmt.Printf("==> provider selected: %s\n", color.HiBlueString("digitalocean"))
+	case "ln", "line", "linode":
+		configProvider = provider.ConfigProvider{
+			Token:        "",
+			Provider:     "linode",
+			DefaultImage: "linode/debian10",
+			Size:         "g6-standard-1",
+			Region:       "us-east",
+			Username:     "root",
+
+			Limit: 0,
+		}
+		fmt.Printf("==> provider selected: %s\n", color.HiBlueString("linode"))
+	case "aw", "asw", "aws":
+		configProvider = provider.ConfigProvider{
+			AccessKeyId:  "",
+			SecretKey:    "",
+			Provider:     "aws",
+			DefaultImage: "ami-0ee39036464b9a87e",
+			Size:         "t2.medium",
+			Region:       "ap-southeast-1",
+			Username:     "admin",
+			Limit:        0,
+		}
+		fmt.Printf("==> provider selected: %s\n", color.HiBlueString("aws"))
+
+	default:
+		configProvider.Provider = "digitalocean"
+		fmt.Printf("==> provider selected: %s\n", color.HiBlueString("digitalocean"))
+	}
+
+	if configProvider.Provider == "aws" {
+		configProvider.AccessKeyId = credentials("AWS AccessKeyId")
+		fmt.Println()
+		configProvider.SecretKey = credentials("AWS SecretKey")
+		fmt.Println()
+	} else {
+		configProvider.Token = credentials("API Token")
+	}
+
+	configProvider.Name = fmt.Sprintf("%s-%s", configProvider.Provider, utils.RandomString(6))
+
+	if configProvider.Provider == "aws" {
+		fmt.Printf("💡 Choose your image carefully because it related to region.\n")
+		fmt.Printf("💡 Refer for this link for more information: " + color.HiCyanString("https://wiki.debian.org/Cloud/AmazonEC2Image/Buster"))
+	}
+	configProvider.DefaultImage = StringPrompt("\n🌀 Choose "+color.HiGreenString("base image")+" for building Osmedeus Image?", configProvider.DefaultImage)
+	configProvider.Size = StringPrompt("🌀 Choose "+color.HiGreenString("instance type")+" for running the scan?", configProvider.Size)
+	configProvider.Region = StringPrompt("🌀 Choose "+color.HiGreenString("instance region")+" for running the scan?", configProvider.Region)
+
+	return configProvider
+}
+
+func credentials(name string) string {
+	fmt.Printf("🔑 Enter your %s? ", color.HiGreenString(name))
+	var token string
+	for {
+		byteToken, err := term.ReadPassword(int(syscall.Stdin))
+		if err == nil && len(byteToken) > 6 {
+			token = strings.TrimSpace(string(byteToken))
+			break
+		}
+		utils.WarnF("Looks like your token is invalid. Please try again: %v", token)
+	}
+
+	redactedToken := token[:5] + "***" + token[len(token)-5:]
+	fmt.Printf("Your data has been saved: %v", color.HiBlueString(redactedToken))
+	return token
+}
+
+// StringPrompt asks for a string value using the label
+func StringPrompt(label string, alt string) string {
+	var s string
+	r := bufio.NewReader(os.Stdin)
+	for {
+		fmt.Fprintf(os.Stderr, fmt.Sprintf("%v (default: %s): ", label, color.HiCyanString(alt)))
+		s, _ = r.ReadString('\n')
+		s = strings.TrimSpace(strings.ToLower(s))
+		if s == "" {
+			if alt != "" {
+				return alt
+			}
+			utils.WarnF("Blank input doesn't allow, please specify one")
+		}
+
+		if s != "" {
+			break
+		}
+	}
+	return strings.TrimSpace(s)
+}
+
+// ValidateProvider setup new provider
+func ValidateProvider(opt libs.Options, providerConfig provider.ConfigProvider) (CloudRunner, error) {
+	var cloudRunner CloudRunner
+	cloudRunner.Opt = opt
+	cloudRunner.Prepare()
+
+	providerCloud, err := provider.InitProviderWithConfig(opt, providerConfig)
+	if err != nil {
+		return cloudRunner, err
+	}
+	cloudRunner.Provider = providerCloud
+
+	// check if snapshot is okay or not
+	if !cloudRunner.Provider.SnapshotFound {
+		utils.InforF("No Snapshot found: %v", cloudRunner.Provider.SnapshotName)
+		err = cloudRunner.Provider.BuildImage()
+		if err != nil {
+			utils.ErrorF("error build snapshot at %v", cloudRunner.Provider.ProviderConfig.BuildFile)
+			return cloudRunner, err
+		}
+	}
+
+	return cloudRunner, nil
+}

execution/clean.go

@@ -0,0 +1,527 @@
+package execution
+
+import (
+	"bufio"
+	"fmt"
+	"net/url"
+	"os"
+	"path"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/flosch/pongo2/v6"
+	"github.com/spf13/cast"
+
+	"github.com/thoas/go-funk"
+
+	"github.com/Jeffail/gabs/v2"
+	"github.com/j3ssie/osmedeus/libs"
+	"github.com/j3ssie/osmedeus/utils"
+)
+
+// Cleaning the execution directory
+func Cleaning(folder string, reports []string) {
+	utils.DebugF("Cleaning result: %v", folder)
+	// list all the file
+	items, err := filepath.Glob(fmt.Sprintf("%v/*", folder))
+	if err != nil {
+		return
+	}
+
+	for _, item := range items {
+		item = utils.NormalizePath(item)
+		utils.DebugF("Check Cleaning: %v", item)
+		if funk.Contains(reports, item) {
+			utils.DebugF("Skip cleaning file: %v", item)
+			continue
+		}
+
+		fi, err := os.Stat(item)
+		if err != nil {
+			continue
+		}
+		switch mode := fi.Mode(); {
+		case mode.IsDir():
+			DeleteFolder(item)
+			continue
+		case mode.IsRegular():
+			DeleteFile(item)
+		}
+	}
+}
+
+// CleanGoBuster clean output for gobuster
+func CleanGoBuster(src string, output string) {
+	data := utils.GetFileContent(src)
+	if data == "" {
+		return
+	}
+	result := strings.Replace(data, "Found: ", "", -1)
+	utils.WriteToFile(output, result)
+}
+
+// CleanMassdns clean result of massdns to get IP address
+func CleanMassdns(filename string, output string) {
+	file, err := os.Open(utils.NormalizePath(filename))
+	if err != nil {
+		return
+	}
+	defer file.Close()
+
+	outputFile, err := os.OpenFile(utils.NormalizePath(output), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
+	if err != nil {
+		return
+	}
+	defer outputFile.Close()
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if line == "" {
+			continue
+		}
+		if strings.Contains(line, " A ") {
+			data := strings.Split(line, " A ")
+			host := strings.Trim(data[0], ".")
+			ip := strings.Trim(data[1], ".")
+			outputFile.WriteString(fmt.Sprintf("%v,%v\n", host, ip))
+		} else if strings.Contains(line, " CNAME ") {
+			data := strings.Split(line, " CNAME ")
+			host := strings.Trim(data[0], ".")
+			ip := strings.Trim(data[1], ".")
+			outputFile.WriteString(fmt.Sprintf("%v,%v\n", host, ip))
+		}
+	}
+}
+
+// CleanAmass get IP range and ASN from Amass result
+func CleanAmass(filename string, output string) {
+	content := utils.ReadingLines(filename)
+	if len(content) <= 0 {
+		return
+	}
+
+	var result []string
+	for _, line := range content {
+		jsonParsed, err := gabs.ParseJSON([]byte(line))
+		if err != nil {
+			continue
+		}
+		asn := jsonParsed.Path("asn").String()
+		cidr := jsonParsed.Path("cidr").String()
+		desc := jsonParsed.Path("desc").String()
+		if cidr != "" && cidr != "null" {
+			result = append(result, fmt.Sprintf("AS%v,%v,%v", asn, cidr, desc))
+		}
+	}
+
+	if len(result) > 0 {
+		utils.WriteToFile(output, strings.Join(result, "\n"))
+	}
+
+}
+
+// CleanSWebanalyze get to get formatted report
+func CleanSWebanalyze(filename string, output string) {
+	content := utils.ReadingLines(filename)
+	if len(content) <= 0 {
+		return
+	}
+
+	result := make(map[string][]string)
+	var finalResult []string
+	for _, line := range content {
+		if strings.TrimSpace(line) == "" {
+			continue
+		}
+		if !strings.Contains(line, "tech~") || !strings.Contains(line, "|") {
+			continue
+		}
+		data := strings.Split(line, "|")
+		domain, _ := utils.GetDomain(strings.TrimLeft(data[1], "~"))
+		tech := data[2]
+		if strings.HasSuffix(tech, "/") {
+			tech = strings.Trim(tech, "/")
+		}
+		result[domain] = append(result[domain], tech)
+	}
+
+	// join summary result
+	for k, v := range result {
+		v = funk.UniqString(v)
+		sort.Strings(v)
+		techs := strings.Join(v, ",")
+		data := fmt.Sprintf("domain|%v;;techs|%v", k, strings.Trim(techs, ","))
+		finalResult = append(finalResult, data)
+	}
+
+	if len(result) > 0 {
+		utils.WriteToFile(output, strings.Join(finalResult, "\n"))
+	}
+}
+
+// CleanWebanalyze get to get formatted report
+func CleanWebanalyze(filename string, output string, sum string) {
+	content := utils.ReadingLines(filename)
+	if len(content) <= 0 {
+		return
+	}
+
+	result := make(map[string][]string)
+	var finalResult []string
+	var techSum []string
+	for _, line := range content {
+		jsonParsed, err := gabs.ParseJSON([]byte(line))
+		if err != nil {
+			continue
+		}
+
+		URL := strings.Trim(jsonParsed.S("hostname").String(), `"`)
+		u, err := url.Parse(URL)
+		if err != nil {
+			continue
+		}
+		matches := jsonParsed.S("matches").Children()
+		for _, child := range matches {
+			techs := ""
+			app := strings.Trim(child.S("app_name").String(), `"`)
+			version := strings.Trim(child.S("version").String(), `"`)
+			if version != "" {
+				techs += fmt.Sprintf("%v/%v,", app, version)
+			} else {
+				techs += fmt.Sprintf("%v,", app)
+			}
+
+			// ignore blank techs
+			if techs == "" || strings.Trim(techs, ",") == "" || strings.Trim(techs, ",") == " " {
+				continue
+			}
+			techSum = append(techSum, strings.Trim(app, ","))
+			techs = strings.TrimSpace(strings.Trim(techs, ","))
+			result[u.Hostname()] = append(result[u.Hostname()], techs)
+		}
+	}
+
+	// join summary result
+	for k, v := range result {
+		techs := strings.Join(funk.UniqString(v), ",")
+		data := fmt.Sprintf("domain|%v;;techs|%v", k, strings.Trim(techs, ","))
+		finalResult = append(finalResult, data)
+	}
+
+	if len(techSum) > 0 {
+		utils.WriteToFile(sum, strings.Join(funk.UniqString(techSum), "\n"))
+	}
+	if len(result) > 0 {
+		utils.WriteToFile(output, strings.Join(finalResult, "\n"))
+	}
+}
+
+// CleanArjun clean output of Arjun
+func CleanArjun(src string, dest string) {
+	src = utils.NormalizePath(src)
+	if !utils.FolderExists(src) {
+		return
+	}
+
+	if !strings.HasSuffix(src, "/") {
+		src += "/"
+	}
+	src = path.Join(src, "*")
+	outputs, err := filepath.Glob(src)
+	if err != nil {
+		return
+	}
+
+	var data []string
+	for _, output := range outputs {
+		content := utils.GetFileContent(output)
+		jsonParsed, err := gabs.ParseJSON([]byte(content))
+		if err != nil {
+			continue
+		}
+		prefix := "[GET]"
+		if strings.HasPrefix(output, "post") {
+			prefix = "[POST]"
+		}
+
+		for URL, child := range jsonParsed.ChildrenMap() {
+			if len(child.Children()) <= 0 {
+				continue
+			}
+			for _, query := range child.Children() {
+				line := fmt.Sprintf("%v %v?%v=FUZZ", prefix, URL, query.Data())
+				data = append(data, line)
+			}
+		}
+	}
+
+	if len(data) > 0 {
+		utils.WriteToFile(dest, strings.Join(data, "\n"))
+	}
+}
+
+// CleanJSONDnsx get to get formatted report
+func CleanJSONDnsx(filename string, dest string) {
+	content := utils.ReadingLines(filename)
+	if len(content) <= 0 {
+		utils.WarnF("File not found: %s", filename)
+		return
+	}
+
+	var results []string
+	for _, line := range content {
+		jsonParsed, err := gabs.ParseJSON([]byte(line))
+		if err != nil {
+			continue
+		}
+
+		domain, ok := jsonParsed.S("host").Data().(string)
+		if !ok {
+			continue
+		}
+
+		a := jsonParsed.S("a")
+		if a != nil {
+			for _, record := range a.Children() {
+				data := fmt.Sprintf("%s A %s", domain, cast.ToString(record.Data()))
+				results = append(results, data)
+			}
+		}
+
+		cname := jsonParsed.S("cname")
+		if cname != nil {
+			for _, record := range cname.Children() {
+				data := fmt.Sprintf("%s CNAME %s", domain, cast.ToString(record.Data()))
+				results = append(results, data)
+			}
+		}
+
+		mx := jsonParsed.S("mx")
+		if mx != nil {
+			for _, record := range mx.Children() {
+				data := fmt.Sprintf("%s MX %s", domain, cast.ToString(record.Data()))
+				results = append(results, data)
+			}
+		}
+
+		ns := jsonParsed.S("ns")
+		if ns != nil {
+			for _, record := range ns.Children() {
+				data := fmt.Sprintf("%s NS %s", domain, cast.ToString(record.Data()))
+				results = append(results, data)
+			}
+		}
+
+	}
+
+	if len(results) > 0 {
+		utils.WriteToFile(dest, strings.Join(results, "\n"))
+	}
+}
+
+// CleanRustScan make rustscan data to flat format ip:port
+func CleanRustScan(src string, dest string) {
+	src = utils.NormalizePath(src)
+	dest = utils.NormalizePath(dest)
+	content := utils.ReadingLines(src)
+	if len(content) <= 0 {
+		utils.WarnF("File not found: %s", src)
+		return
+	}
+
+	var results []string
+	for _, line := range content {
+		// 103.247.207.76 -> [80,80,443,443]
+		if !strings.Contains(line, " -> ") {
+			continue
+		}
+
+		ip := strings.Split(line, " -> ")[0]
+		rPorts := strings.Split(line, " -> ")[1]
+		rPorts = rPorts[1 : len(rPorts)-1]
+
+		if !strings.Contains(rPorts, ",") {
+			results = append(results, fmt.Sprintf("%s:%s", ip, rPorts))
+		}
+
+		ports := strings.Split(rPorts, ",")
+		ports = funk.UniqString(ports)
+		for _, port := range ports {
+			results = append(results, fmt.Sprintf("%s:%s", ip, port))
+		}
+	}
+	utils.WriteToFile(dest, strings.Join(results, "\n"))
+}
+
+type Vulnerability struct {
+	SignID     string
+	SignPath   string
+	URL        string
+	Risk       string
+	Confidence string
+	Request    string
+	ModalID    string
+
+	ReportPath string
+	ReportFile string
+
+	Status string
+	Length string
+	Words  string
+	Time   string
+}
+
+func GenNucleiReport(opt libs.Options, src string, dest string, templateFile string) {
+	if templateFile == "" {
+		templateFile = path.Join(opt.Env.DataFolder, "nuclei-report.html")
+	}
+
+	if !utils.FileExists(src) {
+		utils.WarnF("file not found: %v", src)
+		return
+	}
+	content := utils.ReadingLines(src)
+	var vulns []Vulnerability
+
+	for index, line := range content {
+		if strings.TrimSpace(line) == "" {
+			continue
+		}
+		jsonParsed, err := gabs.ParseJSON([]byte(line))
+		if err != nil {
+			utils.WarnF("Error parse JSON Data")
+			continue
+		}
+
+		modalID, err := utils.GetDomain(cast.ToString(jsonParsed.S("host").Data()))
+		if err != nil {
+			continue
+		}
+		modalID = fmt.Sprintf("%s-%d", strings.ReplaceAll(modalID, ".", "-"), index)
+
+		vulns = append(vulns, Vulnerability{
+			Request:    cast.ToString(jsonParsed.S("request").Data()),
+			URL:        cast.ToString(jsonParsed.S("matched-at").Data()),
+			SignID:     cast.ToString(jsonParsed.S("template-id").Data()),
+			Risk:       cast.ToString(jsonParsed.S("info", "severity").Data()),
+			ModalID:    modalID,
+			Confidence: "Tentative",
+		})
+	}
+
+	utils.DebugF("Reading vuln %v from: %v ", len(vulns), src)
+	if len(vulns) == 0 {
+		utils.WarnF("No Vulnerability found %v", src)
+		return
+	}
+
+	// read template file
+	tmpl := utils.GetFileContent(templateFile)
+	if strings.TrimSpace(tmpl) == "" {
+		utils.WarnF("empty template data: %v", templateFile)
+		return
+	}
+
+	variable := make(map[string]interface{})
+	variable["Title"] = "Nuclei Summary Report"
+	variable["Vulnerabilities"] = vulns
+	variable["CurrentDay"] = utils.GetCurrentDay()
+	variable["Version"] = libs.VERSION
+	variable["Src"] = filepath.Base(src)
+
+	tpl, err := pongo2.FromString(tmpl)
+	if err != nil {
+		utils.WarnF("error render data: %v", err)
+		return
+	}
+	out, ok := tpl.Execute(variable)
+	if ok == nil {
+		utils.DebugF("Writing Nuclei HTML report to: %v", dest)
+		utils.WriteToFile(dest, out)
+	}
+}
+
+// CleanJSONHttpx get to get formatted report
+func CleanJSONHttpx(filename string, dest string) {
+	content := utils.ReadingLines(filename)
+	if len(content) <= 0 {
+		utils.WarnF("File not found: %s", filename)
+		return
+	}
+
+	var results []string
+	for _, line := range content {
+		jsonParsed, err := gabs.ParseJSON([]byte(line))
+		if err != nil {
+			continue
+		}
+
+		URL := cast.ToString(jsonParsed.S("url").Data())
+		//domain, _ := utils.GetDomain(URL)
+		bodyHash := cast.ToString(jsonParsed.S("body-sha256").Data())
+		headerHash := cast.ToString(jsonParsed.S("header-sha256").Data())
+		hash := utils.GenHash(fmt.Sprintf("%s-%s", headerHash, bodyHash))
+
+		title := "No-Title"
+		rawTitle := jsonParsed.S("title")
+		if rawTitle != nil {
+			title = cast.ToString(rawTitle.Data())
+		}
+
+		rawTechs := jsonParsed.S("tech")
+		techs := "No-Tech"
+		if rawTechs != nil {
+			techs = strings.Join(cast.ToStringSlice(rawTechs.Data()), ";;")
+
+		}
+
+		data := fmt.Sprintf("%s,%s,%s,%s", URL, hash, title, techs)
+		results = append(results, data)
+
+	}
+
+	if len(results) > 0 {
+		utils.WriteToFile(dest, strings.Join(results, "\n"))
+	}
+}
+
+// CleanFFUFJson get to get formatted report
+func CleanFFUFJson(filename string, dest string) {
+	content := utils.ReadingLines(filename)
+	if len(content) <= 0 {
+		utils.WarnF("File not found: %s", filename)
+		return
+	}
+
+	var results []string
+	for _, line := range content {
+		jsonParsed, err := gabs.ParseJSON([]byte(line))
+		if err != nil {
+			continue
+		}
+
+		resultsJson := jsonParsed.S("results")
+		if resultsJson == nil {
+			continue
+		}
+
+		for _, item := range resultsJson.Children() {
+			//.url,.status,.length,.words,.lines,.redirectlocation
+			endpoint := cast.ToString(item.S("url").Data())
+			status := cast.ToString(item.S("status").Data())
+			length := cast.ToString(item.S("length").Data())
+			words := cast.ToString(item.S("words").Data())
+			lines := cast.ToString(item.S("lines").Data())
+			redirectLocation := cast.ToString(item.S("redirectlocation").Data())
+
+			data := fmt.Sprintf("%s,%s,%s,%s,%s,%s", endpoint, status, length, words, lines, redirectLocation)
+			results = append(results, data)
+		}
+	}
+
+	if len(results) > 0 {
+		utils.WriteToFile(dest, strings.Join(results, "\n"))
+	}
+}

execution/clean_test.go

@@ -0,0 +1,19 @@
+package execution
+
+//
+//func TestCleanWebanalyze(t *testing.T) {
+//	var options libs.Options
+//	options.Debug = true
+//	CleanWebanalyze("/Users/j3ssie/.osmedeus/workspaces/duckduckgo.com/fingerprint/duckduckgo.com-technology.json", "/Users/j3ssie/.osmedeus/workspaces/duckduckgo.com/fingerprint/duckduckgo.com-technologies.txt")
+//
+//	if !utils.FileExists("/Users/j3ssie/.osmedeus/workspaces/duckduckgo.com/fingerprint/duckduckgo.com-technologies.txt") {
+//		t.WarnF("Error CleanWebanalyze")
+//	}
+//}
+
+//func TestCleanSWebanalyze(t *testing.T) {
+//    CleanSWebanalyze("/tmp/mmm", "/tmp/technologies.txt")
+//    if !utils.FileExists("/tmp/technologies.txt") {
+//        t.WarnF("Error CleanSWebanalyze")
+//    }
+//}

execution/git.go

@@ -0,0 +1,183 @@
+package execution
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/j3ssie/osmedeus/libs"
+	"github.com/j3ssie/osmedeus/utils"
+)
+
+/*
+
+Base Git command:
+GIT_SSH_COMMAND='ssh -o StrictHostKeyChecking=no -i /path/to/private/key' git xxx
+
+*/
+
+/*
+GitDiff('path-to-diff-file', 'path-to-output')
+GitDiff('path-to-diff-folder', 'path-to-output')
+GitDiff('path-to-diff-file', 'path-to-output', 'number-of-commit-to-diff')
+*/
+
+// GitDiff run git diff command
+func GitDiff(dest string, output string, history string, options libs.Options) {
+	if options.NoGit || options.Storages["secret_key"] == "" {
+		return
+	}
+	if !utils.FileExists(dest) {
+		utils.WarnF("File not found: %v", dest)
+		return
+	}
+	utils.DebugF("Git Diff: %v", dest)
+	diffCmd := fmt.Sprintf("GIT_SSH_COMMAND='ssh -o StrictHostKeyChecking=no -i %v' git diff -U0 HEAD~%v --output=%v %v", options.Storages["secret_key"], history, output, dest)
+	Execution(diffCmd, options)
+}
+
+// LoopGitDiff like GitDiff but take input as a file
+func LoopGitDiff(src string, output string, options libs.Options) {
+	lines := utils.ReadingFileUnique(src)
+	if len(lines) == 0 {
+		return
+	}
+	for _, line := range lines {
+		GitDiff(line, output, "1", options)
+	}
+}
+
+// DiffCompare run git diff command
+func DiffCompare(src string, dest string, output string, options libs.Options) {
+	if options.NoGit || options.Storages["secret_key"] == "" {
+		return
+	}
+	// if !utils.FileExists(src) || !utils.FileExists(dest) {
+	// 	return
+	// }
+	diffCmd := fmt.Sprintf("GIT_SSH_COMMAND='ssh -o StrictHostKeyChecking=no -i %v' git diff --no-index --output=%v %v %v", options.Storages["secret_key"], output, src, dest)
+	Execution(diffCmd, options)
+}
+
+// PullResult pull latest data from result repo
+func PullResult(storageFolder string, options libs.Options) {
+	if options.NoGit || options.Storages["secret_key"] == "" {
+		return
+	}
+	if !utils.FolderExists(storageFolder) {
+		return
+	}
+	utils.DebugF("git pull on: %v", storageFolder)
+	pullCmd := fmt.Sprintf("GIT_SSH_COMMAND='ssh -o StrictHostKeyChecking=no -i %v' git -C %v pull -f", options.Storages["secret_key"], storageFolder)
+	Execution(pullCmd, options)
+}
+
+// PushResult push result to git repo
+func PushResult(storageFolder string, commitMess string, options libs.Options) {
+	if options.NoGit || options.Storages["secret_key"] == "" {
+		return
+	}
+
+	if !utils.FolderExists(storageFolder) {
+		return
+	}
+	utils.DebugF("git push on: %v", storageFolder)
+	cmds := []string{
+		fmt.Sprintf(`GIT_SSH_COMMAND='ssh -o StrictHostKeyChecking=no -i %v' git -C %v add -A`, options.Storages["secret_key"], storageFolder),
+		fmt.Sprintf(`GIT_SSH_COMMAND='ssh -o StrictHostKeyChecking=no -i %v' git -C %v commit -m "%v"`, options.Storages["secret_key"], storageFolder, commitMess),
+		fmt.Sprintf(`GIT_SSH_COMMAND='ssh -o StrictHostKeyChecking=no -i %v' git -C %v push -f`, options.Storages["secret_key"], storageFolder),
+		fmt.Sprintf(`GIT_SSH_COMMAND='ssh -o StrictHostKeyChecking=no -i %v' git -C %v push -f`, options.Storages["secret_key"], storageFolder),
+	}
+	// really run git command
+	for _, cmd := range cmds {
+		Execution(cmd, options)
+	}
+}
+
+// GitClone update latest UI and Plugins from default repo
+func GitClone(url string, dest string, forced bool, options libs.Options) {
+	if options.NoGit || options.Storages["secret_key"] == "" {
+		utils.WarnF("Storage Disable")
+		return
+	}
+
+	if url == "" || dest == "" {
+		utils.WarnF("Invalid repo or no destination")
+		return
+	}
+
+	// check if folder is exist or not
+	dest = utils.NormalizePath(dest)
+	if forced {
+		utils.DebugF("Remove: %v", dest)
+		os.RemoveAll(dest)
+	}
+
+	// if folder exist and have .git folder in it, do git pull instead
+	if utils.FolderExists(dest) {
+		if utils.FileExists(path.Join(dest, "/.git/HEAD")) {
+			pullCmd := fmt.Sprintf("GIT_SSH_COMMAND='ssh -o StrictHostKeyChecking=no -i %v' git -C %v pull -f", options.Storages["secret_key"], dest)
+			Execution(pullCmd, options)
+			return
+		}
+	}
+
+	// cloning new
+	utils.DebugF("Cloning: %v", url)
+	cloneCmd := fmt.Sprintf("GIT_SSH_COMMAND='ssh -o StrictHostKeyChecking=no -i %v' git clone --depth=1 %v %v", options.Storages["secret_key"], url, dest)
+	Execution(cloneCmd, options)
+}
+
+// CloneRepo clone the repo
+func CloneRepo(url string, dest string, options libs.Options) error {
+	if !ValidGitURL(url) {
+		return errors.New("invalid repo name")
+	}
+	if options.NoGit || options.Storages["secret_key"] == "" {
+		return errors.New("storage Disable")
+	}
+
+	if url == "" || dest == "" {
+		return errors.New("storage didn't setup correctly")
+	}
+	// check if folder is exist or not
+	dest = utils.NormalizePath(dest)
+	utils.DebugF("Cloning: %v", url)
+
+	if utils.FolderExists(dest) {
+		if utils.FileExists(path.Join(dest, "/.git/HEAD")) {
+			return nil
+		}
+		utils.DebugF("Remove: %v", dest)
+		os.RemoveAll(dest)
+	}
+
+	// cloning new
+	cloneCmd := fmt.Sprintf("GIT_SSH_COMMAND='ssh -o StrictHostKeyChecking=no -i %v' git clone --depth=1 %v %v", options.Storages["secret_key"], url, dest)
+	Execution(cloneCmd, options)
+	if utils.FolderExists(dest) {
+		if utils.FileExists(path.Join(dest, "/.git/HEAD")) {
+			return nil
+		}
+		return nil
+	}
+	return errors.New("fail to clone Repo")
+}
+
+// ValidGitURL simple validate git repo
+func ValidGitURL(raw string) bool {
+	if strings.TrimSpace(raw) == "" {
+		return false
+	}
+	if !strings.HasPrefix(raw, "git@") {
+		return false
+	}
+
+	if !strings.Contains(raw, "github.com") && !strings.Contains(raw, "gitlab.com") {
+		return false
+	}
+
+	return true
+}

execution/git_test.go

@@ -0,0 +1,25 @@
+package execution
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/j3ssie/osmedeus/libs"
+	"github.com/j3ssie/osmedeus/utils"
+)
+
+func TestDiffCompare(t *testing.T) {
+	var options libs.Options
+	options.Debug = true
+	DiffCompare("/Users/j3ssie/.osmedeus/workspaces/duckduckgo.com/subdomain/final-duckduckgo.com.txt", "/Users/j3ssie/.osmedeus/storages/summary/duckduckgo.com/subdomain-duckduckgo.com.txt", "/Users/j3ssie/.osmedeus/workspaces/duckduckgo.com/subdomain/diff-duckduckgo.com-2019-12-28_3:20:9.txt", options)
+	// DiffCompare(options)
+	// fmt.Println(result)
+
+	data := utils.GetFileContent("/Users/j3ssie/.osmedeus/workspaces/duckduckgo.com/subdomain/diff-duckduckgo.com-2019-12-28_3:20:9.txt")
+	fmt.Println(data)
+
+	if !utils.FileExists("/Users/j3ssie/.osmedeus/workspaces/duckduckgo.com/subdomain/diff-duckduckgo.com-2019-12-28_3:20:9.txt") {
+		t.Logf("Error DiffCompare")
+	}
+
+}

execution/gitlab.go

@@ -0,0 +1,206 @@
+package execution
+
+import (
+	"errors"
+	"fmt"
+	"github.com/j3ssie/osmedeus/libs"
+	"github.com/j3ssie/osmedeus/utils"
+	"log"
+	"strings"
+
+	"github.com/xanzy/go-gitlab"
+)
+
+// GitlabAuth do authentication with gitlab
+func GitlabAuth(options libs.Options) (*gitlab.Client, error) {
+	if options.NoGit {
+		return nil, nil
+	}
+	// prefer using username
+	if options.Git.Username != "GITLAB_USER" {
+		utils.DebugF("Do authen with user: %s", options.Git.Username)
+		git, err := gitlab.NewBasicAuthClient(
+			options.Git.Username,
+			options.Git.Password,
+			gitlab.WithBaseURL(options.Git.BaseURL),
+		)
+		if err != nil {
+			utils.WarnF("Error authen in Gitlab %v: %v", options.Git.BaseURL, err)
+			return nil, err
+		}
+		return git, nil
+	}
+
+	if options.Git.Token == "" {
+		return nil, errors.New("no credentials provided")
+	}
+	utils.DebugF("Do authen with token: %s", options.Git.Token)
+	git, err := gitlab.NewClient(
+		options.Git.Token,
+		gitlab.WithBaseURL(options.Git.BaseURL),
+	)
+	if err != nil {
+		utils.WarnF("Error authen in Gitlab %v: %v", options.Git.BaseURL, err)
+		return nil, err
+	}
+	return git, nil
+}
+
+// CreateGitlabRepo create gitlab repo
+func CreateGitlabRepo(repo string, tag string, options libs.Options) string {
+	if options.NoGit {
+		return ""
+	}
+	git, err := GitlabAuth(options)
+	if err != nil {
+		return ""
+	}
+
+	utils.InforF("Create repo: %v", repo)
+	tags := []string{options.Git.DefaultTag}
+	if tag != "" {
+		if strings.Contains(tag, ",") {
+			tags = append(tags, strings.Split(tag, ",")...)
+		} else {
+			tags = append(tags, tag)
+		}
+	}
+
+	opt := gitlab.CreateProjectOptions{
+		Name:          gitlab.String(repo),
+		Path:          nil,
+		DefaultBranch: gitlab.String("master"),
+		Visibility:    gitlab.Visibility(gitlab.PrivateVisibility),
+		TagList:       &tags,
+	}
+	gRepo, _, err := git.Projects.CreateProject(&opt, nil)
+	if err != nil {
+		utils.WarnF("Error to create %v: %v", repo, err)
+		return ""
+	}
+
+	optMember := gitlab.AddProjectMemberOptions{
+		UserID:      gitlab.Int(options.Git.DefaultUID),
+		AccessLevel: gitlab.AccessLevel(gitlab.MaintainerPermissions),
+	}
+	git.ProjectMembers.AddProjectMember(gRepo.ID, &optMember)
+	utils.InforF("Created repo at: %s with pid: %v", gRepo.SSHURLToRepo, gRepo.ID)
+	return gRepo.SSHURLToRepo
+}
+
+// DeleteRepo delete repo by id or name
+func DeleteRepo(repo string, pid int, options libs.Options) {
+	if options.NoGit {
+		return
+	}
+	git, err := GitlabAuth(options)
+	if err != nil {
+		return
+	}
+	// select pid first
+	if pid != 0 {
+		utils.InforF("Delete repo with id: %v", pid)
+		git.Projects.DeleteProject(pid)
+		return
+	}
+
+	user, _, err := git.Users.CurrentUser()
+	projects, _, err := git.Projects.ListUserProjects(user.ID, nil)
+	for _, project := range projects {
+		if project.Name == repo {
+			utils.InforF("Delete repo: %v", repo)
+			git.Projects.DeleteProject(project.ID)
+			return
+		}
+	}
+	utils.WarnF("Project not found: %v", repo)
+}
+
+// ListProjects delete repo by id or name
+func ListProjects(gitUser int, options libs.Options) {
+	if options.NoGit {
+		return
+	}
+	git, err := GitlabAuth(options)
+	if err != nil {
+		utils.WarnF("Err get do authen user: %v", err)
+		return
+	}
+	uid := gitUser
+	var username string
+	if gitUser == 0 {
+		user, _, err := git.Users.CurrentUser()
+		if err != nil {
+			utils.WarnF("Err get current user: %v", err)
+			return
+		}
+		uid = user.ID
+		username = user.Username
+	} else {
+		//user, _, err := git.Users.GetUser(uid, git.Users.)
+		//if err != nil {
+		//	utils.WarnF("Err get current user: %v", err)
+		//	return
+		//}
+		//username = user.Username
+	}
+	utils.InforF("Listing projects of uid: %v", uid)
+	projects, _, err := git.Projects.ListUserProjects(uid, nil)
+	if err != nil {
+		utils.WarnF("Error listing projects: %v", err)
+		return
+	}
+	for _, project := range projects {
+		fmt.Printf("%30s     --   %10d\n", fmt.Sprintf("%s/%s", username, project.Name), project.ID)
+	}
+}
+
+// This example shows how to create a client with username and password.
+func GitAuthSample() {
+	// git, err := gitlab.NewBasicAuthClient(
+	// 	"user",
+	// 	"password",
+	// 	gitlab.WithBaseURL("https://gitlab.com"),
+	// )
+	git, err := gitlab.NewClient(
+		"token-here-s",
+		gitlab.WithBaseURL("https://gitlab.com"),
+	)
+
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// List all projects
+	user, _, err := git.Users.CurrentUser()
+	//spew.Dump(user)
+
+	// Create
+	projects, _, err := git.Projects.ListUserProjects(user.ID, nil)
+	////git.Projects.
+	//group, _, _ := git.Groups.ListAllGroupMembers(9182310, nil)
+	//spew.Dump(group)
+	//tags := []string{"osm", "test"}
+	////git.Projects.ListUserProjects()
+	//opt := gitlab.CreateProjectOptions{
+	//	Name:                        gitlab.String("test-project-tag-osm"),
+	//	Path:                        nil,
+	//	DefaultBranch:               gitlab.String("master"),
+	//	//GroupWithProjectTemplatesID: gitlab.Int(9182310),
+	//	Visibility:                  gitlab.Visibility(gitlab.PrivateVisibility),
+	//	TagList:                     &tags,
+	//}
+	//git.Projects.CreateProject(&opt, nil)
+
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	//git.Projects.DeleteProject(pid)
+
+	log.Printf("Found %d projects", len(projects))
+	for _, project := range projects {
+		fmt.Printf("%30s   --   %10d\n", fmt.Sprintf("%s/%s", user.Username, project.Name), project.ID)
+	}
+
+}

execution/gitlab_test.go

@@ -0,0 +1,9 @@
+package execution
+
+import (
+	"testing"
+)
+
+func TestGitAuth(t *testing.T) {
+	GitAuthSample()
+}

execution/noti.go

@@ -0,0 +1,352 @@
+package execution
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"math/rand"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/spf13/cast"
+
+	tgbotapi "github.com/go-telegram-bot-api/telegram-bot-api/v5"
+	"github.com/j3ssie/osmedeus/libs"
+	"github.com/j3ssie/osmedeus/utils"
+	"github.com/robertkrimen/otto"
+	"github.com/slack-go/slack"
+)
+
+// StatusNoti send to when module is done with report file
+func StatusNoti(notiType string, options libs.Options) {
+	SendAttachment(notiType, "", options)
+}
+
+// ReportNoti send to notification when module is done with report file
+func ReportNoti(arguments []otto.Value, options libs.Options) {
+	if len(arguments) >= 1 {
+		for _, argument := range arguments {
+			SendFile(argument.String(), options.Noti.SlackReportChannel, options)
+		}
+		return
+	}
+	// if doesn't provide report file send all file in noti section
+	for _, file := range options.Module.Report.Noti {
+		SendFile(file, options.Noti.SlackReportChannel, options)
+	}
+}
+
+// DiffNoti send to notification based on diff content
+func DiffNoti(arguments []otto.Value, options libs.Options) {
+	if len(arguments) >= 1 {
+		for _, argument := range arguments {
+			filename := argument.String()
+			if !utils.FileExists(filename) {
+				continue
+			}
+			data := getNewContent(utils.ReadingLines(filename))
+			if strings.TrimSpace(data) == "" {
+				continue
+			}
+			// messageContent := fmt.Sprintf("%v \n ```%v```", filename, data)
+			// SendAttachment("diff", messageContent, options)
+			SendFile(filename, options.Noti.SlackDiffChannel, options)
+		}
+		return
+	}
+	// if doesn't provide report file send all file in noti section
+	for _, filename := range options.Module.Report.Diff {
+		if !utils.FileExists(filename) {
+			continue
+		}
+		data := getNewContent(utils.ReadingLines(filename))
+		if strings.TrimSpace(data) == "" {
+			continue
+		}
+		// messageContent := fmt.Sprintf("%v \n ```%v```", filename, data)
+		// SendAttachment("diff", messageContent, options)
+		SendFile(filename, options.Noti.SlackDiffChannel, options)
+	}
+}
+
+func getNewContent(data []string) string {
+	var result string
+	for _, line := range data {
+		if strings.HasPrefix(line, "+") && !strings.HasPrefix(line, "++") {
+			result += fmt.Sprintf("%v\n", line)
+		}
+	}
+	return result
+}
+
+// SendAttachment send attach message to specific channel
+func SendAttachment(messType string, messContent string, options libs.Options) error {
+	if options.Noti.SlackToken == "" {
+		return errors.New("Slack config improperly")
+	}
+
+	// choose method
+	var channel, color, mess string
+	switch messType {
+	case "start":
+		channel = options.Noti.SlackStatusChannel
+		color = "#005b9f"
+		mess = fmt.Sprintf("%v Start to run *%v* on *%v*", GetEmoji(), options.Module.Name, options.Scan.ROptions["Workspace"])
+		break
+	case "done":
+		channel = options.Noti.SlackStatusChannel
+		color = "#32cb00"
+		mess = fmt.Sprintf("%v Done run *%v* on *%v*", GetEmoji(), options.Module.Name, options.Scan.ROptions["Workspace"])
+		break
+	case "diff":
+		channel = options.Noti.SlackDiffChannel
+		color = "#5E35B1"
+		mess = fmt.Sprintf("%v Diff content on %v: \n %v", GetEmoji(), options.Scan.ROptions["Workspace"], messContent)
+		break
+	case "custom":
+		channel = options.Noti.SlackStatusChannel
+		color = "#1ABC9C"
+		mess = messContent
+		break
+	}
+
+	if channel == "" || mess == "" {
+		return errors.New("Slack channel config improperly")
+	}
+	utils.DebugF("Sending %v message to %v", messType, channel)
+
+	api := slack.New(options.Noti.SlackToken)
+	// message config
+	attachment := slack.Attachment{
+		Color: color,
+		Text:  mess,
+		// sender name
+		Footer:     options.Noti.ClientName,
+		FooterIcon: GetIcon(),
+		Ts:         json.Number(strconv.FormatInt(time.Now().Unix(), 10)),
+	}
+
+	_, _, err := api.PostMessage(channel, slack.MsgOptionAttachments(attachment))
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// SlackWebHook send message with webhook
+func SlackWebHook(webhookURL string, content string) error {
+	content = fmt.Sprintf("```%s```", content)
+	attachment := slack.Attachment{
+		Color: "#1ABC9C",
+		Text:  content,
+		// sender name
+		//Footer:     options.Noti.ClientName,
+		FooterIcon: GetIcon(),
+		Ts:         json.Number(strconv.FormatInt(time.Now().Unix(), 10)),
+	}
+
+	msg := slack.WebhookMessage{
+		Attachments: []slack.Attachment{attachment},
+	}
+	err := slack.PostWebhook(webhookURL, &msg)
+	return err
+}
+
+// WebHookSendAttachment send attach message to specific channel
+func WebHookSendAttachment(options libs.Options, messType string, messContent string) error {
+	if options.NoNoti {
+		return fmt.Errorf("noti disabled")
+	}
+	if options.Noti.SlackWebHook == "" {
+		return errors.New("slack webhook config improperly")
+	}
+
+	// choose method
+	var color, mess string
+	switch messType {
+	case "start":
+		color = "#005b9f"
+		mess = fmt.Sprintf("%v Start to run *%v* on *%v*", GetEmoji(), options.Module.Name, options.Scan.ROptions["Workspace"])
+		break
+	case "done":
+		color = "#32cb00"
+		mess = fmt.Sprintf("%v Done run *%v* on *%v*", GetEmoji(), options.Module.Name, options.Scan.ROptions["Workspace"])
+		break
+	case "diff":
+		color = "#5E35B1"
+		mess = fmt.Sprintf("%v Diff content on %v: \n %v", GetEmoji(), options.Scan.ROptions["Workspace"], messContent)
+		break
+	case "custom":
+		color = "#1ABC9C"
+		mess = messContent
+		break
+	default:
+		color = "#1ABC9C"
+		mess = messContent
+	}
+
+	// message config
+	attachment := slack.Attachment{
+		Color: color,
+		Text:  mess,
+		// sender name
+		Footer:     options.Noti.ClientName,
+		FooterIcon: GetIcon(),
+		Ts:         json.Number(strconv.FormatInt(time.Now().Unix(), 10)),
+	}
+
+	msg := slack.WebhookMessage{
+		Attachments: []slack.Attachment{attachment},
+	}
+	err := slack.PostWebhook(options.Noti.SlackWebHook, &msg)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// SendFile send file to specific channel
+func SendFile(filename string, channel string, options libs.Options) error {
+	if options.Noti.SlackToken == "" || options.Noti.SlackReportChannel == "" {
+		return fmt.Errorf("Slack config improperly")
+	}
+
+	if !utils.FileExists(filename) {
+		return fmt.Errorf("report file not found: %v", filename)
+	}
+
+	baseName := filepath.Base(filename)
+	mess := fmt.Sprintf("%v - %v - Report file for *%v* on *%v*", GetEmoji(), baseName, options.Module.Name, options.Scan.ROptions["Workspace"])
+	utils.DebugF("Sending %v message to %v", filename, channel)
+
+	// sending file
+	api := slack.New(options.Noti.SlackToken)
+	params := slack.FileUploadParameters{
+		Channels: []string{channel},
+		Title:    mess,
+		Filetype: "txt",
+		File:     filename,
+	}
+	_, err := api.UploadFile(params)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// TeleSendMess send message to telegram
+func TeleSendMess(options libs.Options, content string, channel string, wrap bool) error {
+
+	if options.NoNoti {
+		return fmt.Errorf("noti disabled")
+	}
+	bot, err := tgbotapi.NewBotAPI(options.Noti.TelegramToken)
+	content = tgbotapi.EscapeText(tgbotapi.ModeMarkdown, content)
+	if wrap {
+		content = fmt.Sprintf("```\n%s\n```", content)
+	}
+	if err != nil {
+		utils.DebugF("error init telegram: %v", err)
+		return err
+	}
+
+	if channel == "" || channel == "general" {
+		channel = options.Noti.TelegramChannel
+	}
+	switch channel {
+	case "#status":
+		channel = options.Noti.TelegramStatusChannel
+	case "#r", "#report", "#reports", "#vuln":
+		channel = options.Noti.TelegramReportChannel
+	case "#s", "#sensitive", "#sen":
+		channel = options.Noti.TelegramSensitiveChannel
+	case "#dirb", "#dirscan":
+		channel = options.Noti.TelegramDirbChannel
+	case "#m", "#mics":
+		channel = options.Noti.TelegramMicsChannel
+	case "#default", "#general":
+		channel = options.Noti.TelegramChannel
+	}
+	telechannel := cast.ToInt64(channel)
+	utils.DebugF("send message to channel %v", channel)
+	msg := tgbotapi.NewMessage(telechannel, content)
+	msg.ParseMode = "markdown"
+	_, err = bot.Send(msg)
+	if err != nil {
+		utils.DebugF("error sending telegram to %v -- %v", channel, err)
+	}
+	return err
+}
+
+// TeleSendFile send message to telegram
+func TeleSendFile(options libs.Options, filename string, channel string) error {
+	if options.NoNoti {
+		return fmt.Errorf("noti disabled")
+	}
+	bot, err := tgbotapi.NewBotAPI(options.Noti.TelegramToken)
+	if err != nil {
+		utils.DebugF("error init telegram: %v", err)
+		return err
+	}
+
+	if channel == "" || channel == "general" {
+		channel = options.Noti.TelegramChannel
+	}
+	switch channel {
+	case "#status":
+		channel = options.Noti.TelegramStatusChannel
+	case "#r", "#report", "#reports", "#vuln":
+		channel = options.Noti.TelegramReportChannel
+	case "#sensitive", "#sen":
+		channel = options.Noti.TelegramSensitiveChannel
+	case "#dirb", "#dirscan":
+		channel = options.Noti.TelegramDirbChannel
+	case "#m", "#mics":
+		channel = options.Noti.TelegramMicsChannel
+	case "#default":
+		channel = options.Noti.TelegramChannel
+	}
+	telechannel := cast.ToInt64(channel)
+
+	filename = utils.NormalizePath(filename)
+
+	msg := tgbotapi.NewDocument(telechannel, tgbotapi.FilePath(filename))
+	utils.DebugF("send file %v to channel %v", filename, channel)
+	_, err = bot.Send(msg)
+	if err != nil {
+		utils.DebugF("error sending telegram to %v -- %v", channel, err)
+	}
+	return err
+}
+
+/////// utils for slack message
+
+// GetEmoji get random emoji
+func GetEmoji() string {
+	rand.Seed(time.Now().Unix())
+	emojis := []string{
+		":robot_face:",
+		":alien:",
+		":gift:",
+		":gun:",
+		":diamond_shape_with_a_dot_inside:",
+		":rocket:",
+		":bug:",
+		":broccoli:",
+		":shamrock:",
+	}
+	n := rand.Int() % len(emojis)
+	return emojis[n]
+}
+
+// GetIcon get random emoji
+func GetIcon() string {
+	rand.Seed(time.Now().Unix())
+	emojis := []string{
+		"https://platform.slack-edge.com/img/default_application_icon.png",
+	}
+	n := rand.Int() % len(emojis)
+	return emojis[n]
+}

execution/noti_test.go

@@ -0,0 +1,44 @@
+package execution
+
+import (
+	"github.com/j3ssie/osmedeus/libs"
+	"testing"
+)
+
+func TestSlackWebHook(t *testing.T) {
+
+	content := `
+[jira-subversion-xss][Tentative-Medium] - https://142.104.128.133:443/plugins/servlet/svnwebclient/error.jsp?errormessage=%27%22%3E%3Cscript%3Ealert(document.domain)%3C%2Fscript%3E&description=test - out/142.104.128.133/jira-subversion-xss-b38aaad1bc0567262e48e374e902bda60d522f94\n
+[jira-subversion-xss][Tentative-Medium] - http://58.49.154.201:8080/plugins/servlet/svnwebclient/error.jsp?errormessage=%27%22%3E%3Cscript%3Ealert(document.domain)%3C%2Fscript%3E&description=test - out/58.49.154.201/jira-subversion-xss-2e18d80689f97485af742d8c0368ffd2cdc78d7a
+`
+	err := SlackWebHook("https://hooks.slack.com/services/T01D9M3RSLA/B01CUUVD98X/xxxxxxxx", content)
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func TestTeleSendMess(t *testing.T) {
+	var opt libs.Options
+	opt.Noti.TelegramChannel = "-1001166523435"
+	opt.Noti.TelegramToken = "1288534500:xxxxxx"
+
+	content := `
+[jira-subversion-xss][Tentative-Medium] - https://142.104.128.133:443/plugins/servlet/svnwebclient/error.jsp?errormessage=%27%22%3E%3Cscript%3Ealert(document.domain)%3C%2Fscript%3E&description=test - out/142.104.128.133/jira-subversion-xss-b38aaad1bc0567262e48e374e902bda60d522f94\n
+[jira-subversion-xss][Tentative-Medium] - http://58.49.154.201:8080/plugins/servlet/svnwebclient/error.jsp?errormessage=%27%22%3E%3Cscript%3Ealert(document.domain)%3C%2Fscript%3E&description=test - out/58.49.154.201/jira-subversion-xss-2e18d80689f97485af742d8c0368ffd2cdc78d7a
+`
+	err := TeleSendMess(opt, content, "general", true)
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func TestTeleSendFile(t *testing.T) {
+	var opt libs.Options
+	opt.Noti.TelegramChannel = "-1001166523435"
+	opt.Noti.TelegramToken = "1288534500:xxxx"
+
+	err := TeleSendFile(opt, "/tmp/jtt/out/jaeles-summary.txt", "general")
+	if err != nil {
+		t.Error(err)
+	}
+}

execution/process.go

@@ -0,0 +1,93 @@
+package execution
+
+import (
+	"fmt"
+	"strings"
+	"sync"
+
+	"github.com/fatih/color"
+	"github.com/j3ssie/osmedeus/libs"
+	gops "github.com/mitchellh/go-ps"
+	"github.com/panjf2000/ants"
+	"github.com/shirou/gopsutil/process"
+	"github.com/spf13/cast"
+)
+
+func ListAllOsmedeusProcess() (pids []int) {
+	processes, err := gops.Processes()
+	if err != nil {
+		return pids
+	}
+	var allProcess []OSProcess
+
+	var wg sync.WaitGroup
+	p, _ := ants.NewPoolWithFunc(20, func(i interface{}) {
+		defer wg.Done()
+
+		ps := i.(gops.Process)
+		pid := ps.Pid()
+		proc, _ := process.NewProcess(cast.ToInt32(pid))
+		cmd, _ := proc.Cmdline()
+		if !strings.Contains(cmd, libs.BINARY) {
+			return
+		}
+
+		osProcess := OSProcess{
+			PID:     pid,
+			Command: cmd,
+		}
+		fmt.Printf("pid:%v %s %v\n", color.HiCyanString("%v", osProcess.PID), color.HiMagentaString("--"), osProcess.Command)
+		allProcess = append(allProcess, osProcess)
+		pids = append(pids, osProcess.PID)
+	}, ants.WithPreAlloc(true))
+	defer p.Release()
+
+	for _, ps := range processes {
+		wg.Add(1)
+		_ = p.Invoke(ps)
+
+	}
+	wg.Wait()
+
+	return pids
+}
+
+type OSProcess struct {
+	PID     int    `json:"pid"`
+	Command string `json:"command"`
+}
+
+func GetOsmProcess(processName string) []OSProcess {
+	if processName == "" {
+		processName = libs.BINARY
+	}
+	var results []OSProcess
+	processes, err := gops.Processes()
+	if err != nil {
+		return results
+	}
+
+	for _, ps := range processes {
+		pid := ps.Pid()
+		binary := ps.Executable()
+
+		if strings.ToLower(binary) != strings.ToLower(processName) {
+			continue
+		}
+
+		proc, _ := process.NewProcess(cast.ToInt32(pid))
+		cmd, _ := proc.Cmdline()
+
+		if strings.Contains(cmd, fmt.Sprintf("%s utils ps", libs.BINARY)) {
+			continue
+		}
+
+		osmProcess := OSProcess{
+			PID:     pid,
+			Command: cmd,
+		}
+		results = append(results, osmProcess)
+	}
+
+	return results
+}

execution/process_test.go

@@ -0,0 +1,9 @@
+package execution
+
+import (
+	"testing"
+)
+
+func TestListProcess(t *testing.T) {
+	ListAllOsmedeusProcess()
+}

execution/remote.go

@@ -0,0 +1,126 @@
+package execution
+
+import (
+	"fmt"
+	"net/url"
+	"path"
+	"strings"
+
+	"github.com/Jeffail/gabs/v2"
+	"github.com/j3ssie/osmedeus/libs"
+	"github.com/j3ssie/osmedeus/utils"
+	"github.com/parnurzeal/gorequest"
+)
+
+var remoteOptions libs.Options
+
+// RemoteLogin get to get JWT to run
+func RemoteLogin(username string, password string, URL string, options libs.Options) {
+	u, err := url.Parse(URL)
+	if err != nil {
+		return
+	}
+	URL = fmt.Sprintf("%v://%v:%v", u.Scheme, u.Hostname(), u.Port())
+
+	request := gorequest.New()
+	_, body, _ := request.Post(fmt.Sprintf("%v/auth/login", URL)).
+		Set("Content-Type", "application/json").
+		Send(fmt.Sprintf(`{"username":"%v", "password":"%v"}`, username, password)).
+		End()
+
+	jsonParsed, _ := gabs.ParseJSON([]byte(body))
+
+	token := strings.Trim(jsonParsed.S("token").String(), `"`)
+	if token == "null" {
+		utils.WarnF("Login fail at %v", URL)
+	}
+	options.Client.JWT = fmt.Sprintf("Osmedeus %v", token)
+	options.Client.URL = URL
+	utils.DebugF("Got token: %v at %v", options.Client.JWT, options.Client.URL)
+	remoteOptions = options
+}
+
+// RemoteUpload upload data to remote url
+func RemoteUpload(src string, options libs.Options) {
+	if options.Client.JWT == "" || options.Client.URL == "" {
+		utils.WarnF("JWT Token was not set")
+		return
+	}
+	// keep \n not escape from JSON body
+	data := strings.Join(utils.ReadingFileUnique(utils.NormalizePath(src)), "\\n")
+	filename := path.Base(src)
+	jsonBody := fmt.Sprintf(`{"data":"%v", "filename":"%v"}`, data, filename)
+
+	request := gorequest.New()
+	_, body, _ := request.Post(fmt.Sprintf("%v/api/upload/data", options.Client.URL)).
+		Set("Content-Type", "application/json").
+		Set("Authorization", options.Client.JWT).
+		Send(jsonBody).
+		End()
+	// sample --> /tmp/data-osm-sample
+	jsonParsed, _ := gabs.ParseJSON([]byte(body))
+	remoteFile := strings.Trim(jsonParsed.S("content").String(), `"`)
+
+	if remoteFile == "null" {
+		utils.WarnF("Fail to Upload %v at %v", src, options.Client.URL)
+	}
+	utils.DebugF("Sucessfully uploaded: %v", remoteFile)
+}
+
+// RemoteExec run command on a remote client
+func RemoteExec(command string, options libs.Options) {
+	if options.Client.JWT == "" || options.Client.URL == "" {
+		utils.WarnF("JWT Token was not set")
+		return
+	}
+
+	// default master password is blank now
+	mpassword := ""
+	request := gorequest.New()
+	_, body, _ := request.Post(fmt.Sprintf("%v/api/task/new", options.Client.URL)).
+		Set("Content-Type", "application/json").
+		Set("Authorization", options.Client.JWT).
+		Send(fmt.Sprintf(`{"command": "%v","password": "%v"}`, command, mpassword)).
+		End()
+
+	jsonParsed, _ := gabs.ParseJSON([]byte(body))
+	message := jsonParsed.S("content").String()
+	if message == "null" {
+		utils.WarnF("Run Command Fail at %v", options.Client.URL)
+	}
+	utils.DebugF("Sucessfully run Command: %v", command)
+
+}
+
+// RemoteExecSchedule run command on a remote client with schedule
+func RemoteExecSchedule(command string, schedule string, options libs.Options) {
+	if options.Client.JWT == "" || options.Client.URL == "" {
+		utils.WarnF("JWT Token was not set")
+		return
+	}
+
+	// default master password is blank now
+	mpassword := ""
+	request := gorequest.New()
+	_, body, _ := request.Post(fmt.Sprintf("%v/api/task/new", options.Client.URL)).
+		Set("Content-Type", "application/json").
+		Set("Authorization", options.Client.JWT).
+		Send(fmt.Sprintf(`{"command":"%v","seconds":%v,"password":"%v"}`, command, schedule, mpassword)).
+		End()
+
+	jsonParsed, _ := gabs.ParseJSON([]byte(body))
+	message := jsonParsed.S("content").String()
+	if message == "null" {
+		utils.WarnF("Run Schedule Command Fail at %v", options.Client.URL)
+	}
+	utils.DebugF("Sucessfully run Schedule Command: %v", command)
+
+	/*
+		{
+			"name": "example",
+		    "command": "id",
+		    "minutes": 1,
+		    "password": "321"
+		}
+	*/
+}

execution/request.go

@@ -0,0 +1,227 @@
+package execution
+
+import (
+	"bufio"
+	"crypto/tls"
+	"encoding/base64"
+	"fmt"
+	"github.com/j3ssie/osmedeus/libs"
+	"github.com/j3ssie/osmedeus/utils"
+	"io"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/go-resty/resty/v2"
+	"github.com/sirupsen/logrus"
+)
+
+func SendGET(token string, url string) (libs.Response, error) {
+	client := BuildClient(token, 2)
+	res, err := JustSend(url, client)
+	return res, err
+}
+
+// BuildClient build base HTTP client
+func BuildClient(token string, retry int) *resty.Client {
+	headers := map[string]string{
+		"UserAgent":    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36",
+		"Content-Type": "application/json",
+	}
+	if token != "" {
+		headers["Authorization"] = fmt.Sprintf("Bearer %s", token)
+	}
+
+	timeout := 15
+
+	//if len(options.Headers) > 0 {
+	//	for _, head := range options.Headers {
+	//		if strings.Contains(head, ":") {
+	//			data := strings.Split(head, ":")
+	//			if len(data) < 2 {
+	//				continue
+	//			}
+	//			headers[data[0]] = strings.Join(data[1:], "")
+	//		}
+	//	}
+	//}
+
+	// disable log when retry
+	logger := logrus.New()
+	logger.Out = io.Discard
+
+	client := resty.New()
+	client.SetLogger(logger)
+	client.SetTransport(&http.Transport{
+		MaxIdleConns:          100,
+		MaxConnsPerHost:       1000,
+		IdleConnTimeout:       time.Duration(timeout) * time.Second,
+		ExpectContinueTimeout: time.Duration(timeout) * time.Second,
+		ResponseHeaderTimeout: time.Duration(timeout) * time.Second,
+		TLSHandshakeTimeout:   time.Duration(timeout) * time.Second,
+		DisableCompression:    true,
+		TLSClientConfig:       &tls.Config{InsecureSkipVerify: true},
+	})
+
+	client.SetHeaders(headers)
+	client.SetCloseConnection(true)
+	client.SetRetryCount(retry)
+	client.SetTimeout(time.Duration(timeout) * time.Second)
+	client.SetRetryWaitTime(time.Duration(timeout/2) * time.Second)
+	client.SetRetryMaxWaitTime(time.Duration(timeout) * time.Second)
+	return client
+}
+
+// JustSend just sending request
+func JustSend(url string, client *resty.Client) (res libs.Response, err error) {
+	method := "GET"
+	// redirect policy
+
+	var resp *resty.Response
+	// really sending things here
+	method = strings.ToLower(strings.TrimSpace(method))
+	switch method {
+	case "get":
+		resp, err = client.R().
+			Get(url)
+		break
+	case "post":
+		resp, err = client.R().
+			Post(url)
+		break
+	}
+
+	// in case we want to get redirect stuff
+	if res.StatusCode != 0 {
+		return res, nil
+	}
+
+	if err != nil || resp == nil {
+		utils.WarnF("%v %v", url, err)
+		return libs.Response{}, err
+	}
+
+	return ParseResponse(*resp), nil
+}
+
+// ParseResponse field to Response
+func ParseResponse(resp resty.Response) (res libs.Response) {
+	// var res libs.Response
+	resLength := len(string(resp.Body()))
+	// format the headers
+	var resHeaders []map[string]string
+	for k, v := range resp.RawResponse.Header {
+		if k == "Content-Type" {
+			res.ContentType = strings.Join(v[:], "")
+		}
+		if k == "Location" {
+			res.Location = strings.Join(v[:], "")
+		}
+		element := make(map[string]string)
+		element[k] = strings.Join(v[:], "")
+		resLength += len(fmt.Sprintf("%s: %s\n", k, strings.Join(v[:], "")))
+		resHeaders = append(resHeaders, element)
+	}
+	// response time in second
+	resTime := float64(resp.Time()) / float64(time.Second)
+	resHeaders = append(resHeaders,
+		map[string]string{"Total Length": strconv.Itoa(resLength)},
+		map[string]string{"Response Time": fmt.Sprintf("%f", resTime)},
+	)
+
+	// set some variable
+	res.Headers = resHeaders
+	res.StatusCode = resp.StatusCode()
+	res.Status = fmt.Sprintf("%v %v", resp.Status(), resp.RawResponse.Proto)
+	res.Body = string(resp.Body())
+	res.ResponseTime = resTime
+	res.Length = resLength
+	// beautify
+	res.Beautify = BeautifyResponse(res)
+	res.BeautifyHeader = BeautifyHeaders(res)
+	return res
+}
+
+// BeautifyRequest beautify request
+func BeautifyRequest(req libs.Request) string {
+	var beautifyReq string
+	// hardcoded HTTP/1.1 for now
+	beautifyReq += fmt.Sprintf("%v %v HTTP/1.1\n", req.Method, req.URL)
+
+	for _, header := range req.Headers {
+		for key, value := range header {
+			if key != "" && value != "" {
+				beautifyReq += fmt.Sprintf("%v: %v\n", key, value)
+			}
+		}
+	}
+	if req.Body != "" {
+		beautifyReq += fmt.Sprintf("\n%v\n", req.Body)
+	}
+	return beautifyReq
+}
+
+// BeautifyHeaders beautify headers
+func BeautifyHeaders(res libs.Response) string {
+	beautifyHeader := fmt.Sprintf("< %v \n", res.Status)
+	for _, header := range res.Headers {
+		for key, value := range header {
+			beautifyHeader += fmt.Sprintf("< %v: %v\n", key, value)
+		}
+	}
+	return beautifyHeader
+}
+
+// BeautifyResponse beautify response
+func BeautifyResponse(res libs.Response) string {
+	var beautifyRes string
+	beautifyRes += fmt.Sprintf("%v \n", res.Status)
+
+	for _, header := range res.Headers {
+		for key, value := range header {
+			beautifyRes += fmt.Sprintf("%v: %v\n", key, value)
+		}
+	}
+
+	beautifyRes += fmt.Sprintf("\n%v\n", res.Body)
+	return beautifyRes
+}
+
+// ParseBurpRequest parse burp style request
+func ParseBurpRequest(raw string) string {
+	rawDecoded, err := base64.StdEncoding.DecodeString(raw)
+	if err != nil {
+		return ""
+	}
+
+	var realReq libs.Request
+	reader := bufio.NewReader(strings.NewReader(string(rawDecoded)))
+	parsedReq, err := http.ReadRequest(reader)
+	if err != nil {
+		return ""
+	}
+	realReq.Method = parsedReq.Method
+	// URL part
+	if parsedReq.URL.Host == "" {
+		realReq.Host = parsedReq.Host
+		parsedReq.URL.Host = parsedReq.Host
+	}
+	if parsedReq.URL.Scheme == "" {
+		if parsedReq.Referer() == "" {
+			realReq.Scheme = "https"
+			parsedReq.URL.Scheme = "https"
+		} else {
+			u, err := url.Parse(parsedReq.Referer())
+			if err == nil {
+				realReq.Scheme = u.Scheme
+				parsedReq.URL.Scheme = u.Scheme
+			}
+		}
+	}
+	realReq.URL = parsedReq.URL.String()
+	realReq.Path = parsedReq.RequestURI
+
+	return realReq.URL
+}

execution/require.go

@@ -0,0 +1,106 @@
+package execution
+
+//
+//// RunRequire is main function for generator
+//func RunRequire(script string, options libs.Options) bool {
+//    // @NOTE: for some reason < auto translate to &lt; in golang template
+//    if strings.Contains(script, "&lt;") {
+//        script = strings.Replace(script, "&lt;", "<", -1)
+//    }
+//    utils.DebugF("[Run-Require] %v", script)
+//    vm := otto.New()
+//
+//    vm.Set("EmptyDir", func(call otto.FunctionCall) otto.Value {
+//        result, _ := vm.ToValue(utils.EmptyDir(call.Argument(0).String()))
+//        return result
+//    })
+//
+//    vm.Set("NotEmptyDir", func(call otto.FunctionCall) otto.Value {
+//        result, _ := vm.ToValue(!utils.EmptyDir(call.Argument(0).String()))
+//        return result
+//    })
+//
+//    vm.Set("NotEmptyFile", func(call otto.FunctionCall) otto.Value {
+//        result, _ := vm.ToValue(!utils.EmptyFile(call.Argument(0).String(), 0))
+//        if len(call.ArgumentList) > 1 {
+//            num, _ := call.Argument(0).ToInteger()
+//            result, _ = vm.ToValue(!utils.EmptyFile(call.Argument(0).String(), int(num)))
+//        }
+//        return result
+//    })
+//
+//    vm.Set("EmptyFile", func(call otto.FunctionCall) otto.Value {
+//        result, _ := vm.ToValue(utils.EmptyFile(call.Argument(0).String(), 0))
+//        if len(call.ArgumentList) > 1 {
+//            num, _ := call.Argument(0).ToInteger()
+//            result, _ = vm.ToValue(utils.EmptyFile(call.Argument(0).String(), int(num)))
+//        }
+//        return result
+//    })
+//
+//    vm.Set("ExecContain", func(call otto.FunctionCall) otto.Value {
+//        var validate bool
+//        args := call.ArgumentList
+//        cmd := args[0].String()
+//        search := args[1].String()
+//        out, _ := Execution(cmd, options)
+//        if strings.Contains(out, search) {
+//            validate = true
+//        }
+//        result, _ := vm.ToValue(validate)
+//        return result
+//    })
+//
+//    vm.Set("ExecMatch", func(call otto.FunctionCall) otto.Value {
+//        var validate bool
+//        args := call.ArgumentList
+//        cmd := args[0].String()
+//        search := args[1].String()
+//        out, _ := Execution(cmd, options)
+//        validate = RegexCount(out, search)
+//        result, _ := vm.ToValue(validate)
+//        return result
+//    })
+//
+//    vm.Set("DirLength", func(call otto.FunctionCall) otto.Value {
+//        validate := utils.DirLength(call.Argument(0).String())
+//        result, err := vm.ToValue(validate)
+//        if err != nil {
+//            return otto.FalseValue()
+//        }
+//        return result
+//    })
+//
+//    vm.Set("FileLength", func(call otto.FunctionCall) otto.Value {
+//        validate := utils.FileLength(call.Argument(0).String())
+//        result, err := vm.ToValue(validate)
+//        if err != nil {
+//            return otto.FalseValue()
+//        }
+//        return result
+//    })
+//
+//    result, serr := vm.Run(script)
+//    if serr != nil {
+//        return false
+//    }
+//    analyzeResult, err := result.Export()
+//    if err != nil || analyzeResult == nil {
+//        return false
+//    }
+//    utils.DebugF("Required: %v -- %v", script, result)
+//    return analyzeResult.(bool)
+//}
+//
+//// RegexCount count regex string in component
+//func RegexCount(component string, analyzeString string) bool {
+//    r, err := regexp.Compile(analyzeString)
+//    if err != nil {
+//        return false
+//    }
+//    matches := r.FindAllStringIndex(component, -1)
+//    if len(matches) > 0 {
+//        return true
+//    }
+//    return false
+//}

execution/s3_cdn.go

@@ -0,0 +1,125 @@
+package execution
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/fatih/color"
+	"github.com/j3ssie/osmedeus/libs"
+	"github.com/j3ssie/osmedeus/utils"
+)
+
+// UploadToS3 upload local file to s3 bucket
+func UploadToS3(options libs.Options, source string, bucket string) {
+	if options.Cdn.AccessKeyId == "CDN_AWS_ACCESS_KEY" {
+		return
+	}
+	utils.InforF("Uploading %s to %s", color.HiCyanString(source), color.HiCyanString(bucket))
+	// Set up a new session and get a reference to the S3 service.
+	sess, err := session.NewSession(&aws.Config{
+		Credentials: credentials.NewStaticCredentials(options.Cdn.AccessKeyId, options.Cdn.SecretKey, ""),
+		Region:      aws.String(options.Cdn.Region)},
+	)
+	if err != nil {
+		utils.ErrorF("error creating session: %s", err)
+		return
+	}
+	svc := s3.New(sess)
+
+	if !utils.FileExists(source) {
+		utils.ErrorF("File not found: %s", source)
+		return
+	}
+
+	// Open the file that you want to upload.
+	file, err := os.Open(source)
+	if err != nil {
+		utils.ErrorF("error opening file: %s", err)
+		return
+	}
+	defer file.Close()
+
+	// Set up the parameters for the upload.
+	params := &s3.PutObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(source),
+		Body:   file,
+	}
+
+	// Upload the file to S3.
+	_, err = svc.PutObject(params)
+	if err != nil {
+		utils.ErrorF("error uploading file: %s", err)
+		return
+	}
+}
+
+// DownloadFromS3 upload local file to s3 bucket
+func DownloadFromS3(options libs.Options, source string, dest string, bucket string) {
+	if options.Cdn.AccessKeyId == "CDN_AWS_ACCESS_KEY" {
+		return
+	}
+	utils.InforF("Downloading %s from %s bucket to %s", color.HiCyanString(source), color.HiCyanString(bucket), color.HiBlueString(dest))
+	// Set up a new session and get a reference to the S3 service.
+	sess, err := session.NewSession(&aws.Config{
+		Credentials: credentials.NewStaticCredentials(options.Cdn.AccessKeyId, options.Cdn.SecretKey, ""),
+		Region:      aws.String(options.Cdn.Region)},
+	)
+	if err != nil {
+		utils.ErrorF("error creating session: %s", err)
+		return
+	}
+	svc := s3.New(sess)
+
+	// Set up the parameters for the download.
+	params := &s3.GetObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(source),
+	}
+
+	// Download the file from S3.
+	result, err := svc.GetObject(params)
+	if err != nil {
+		utils.ErrorF("error downloading file: %s", err)
+		return
+	}
+	defer result.Body.Close()
+
+	// Create a local file to store the downloaded data.
+	outFile, err := os.Create(dest)
+	if err != nil {
+		utils.ErrorF("error creating local file: %s", err)
+		return
+	}
+	defer outFile.Close()
+
+	// Copy the data from the S3 object to the local file.
+	_, err = io.Copy(outFile, result.Body)
+	if err != nil {
+		utils.ErrorF("error copying file data: %s", err)
+		return
+	}
+}
+
+// DownloadFile get file from CDN URL
+func DownloadFile(options libs.Options, downloadURL string, dest string) {
+	utils.DebugF("Downloading: %s", downloadURL)
+	if !utils.FolderExists(path.Dir(dest)) {
+		utils.MakeDir(path.Dir(dest))
+	}
+
+	cmd := fmt.Sprintf("wget -qO %s %s", dest, downloadURL)
+	Execution(cmd, options)
+
+	if utils.FileLength(dest) <= 0 {
+		os.RemoveAll(dest)
+		return
+	}
+
+}

execution/scripts.go

@@ -0,0 +1,225 @@
+package execution
+
+import (
+	"archive/tar"
+	"bufio"
+	"compress/gzip"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path"
+	"regexp"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/j3ssie/osmedeus/libs"
+
+	"github.com/j3ssie/osmedeus/utils"
+)
+
+// Execution Run a command
+func Execution(cmd string, options libs.Options) (string, error) {
+	command := []string{
+		"bash",
+		"-c",
+		cmd,
+	}
+	//var output string
+	utils.DebugF("[Exec] %v", command)
+	realCmd := exec.Command(command[0], command[1:]...)
+	if options.Quite == true {
+		realCmd.CombinedOutput()
+	} else {
+		// output command output to std too
+		cmdReader, _ := realCmd.StdoutPipe()
+		scanner := bufio.NewScanner(cmdReader)
+		//var out string
+		go func() {
+			for scanner.Scan() {
+				utils.InforF(scanner.Text())
+			}
+		}()
+		if err := realCmd.Start(); err != nil {
+			return "", err
+		}
+		if err := realCmd.Wait(); err != nil {
+			return "", err
+		}
+	}
+	return "", nil
+}
+
+// Echo just print testing
+func Echo(script string) {
+	fmt.Println("testing:", script)
+}
+
+// Sleep just print testing
+func Sleep(raw string) {
+	time.Sleep(time.Second * time.Duration(utils.StrToInt(raw)))
+}
+
+// Printf print out some string
+func Printf(block string, content string) {
+	if block != "" {
+		utils.BlockF(block, content)
+	} else {
+		fmt.Println(content)
+	}
+}
+
+// ErrPrintf print out some string
+func ErrPrintf(block string, content string) {
+	if block != "" {
+		utils.BadBlockF(block, content)
+	} else {
+		fmt.Println(content)
+	}
+}
+
+// Base just print testing
+func Base(raw string) string {
+	raw = utils.NormalizePath(raw)
+	return path.Base(raw)
+}
+
+// StripName strip a file name
+func StripName(raw string) string {
+	result := strings.Replace(raw, "/", "_", -1)
+	result = strings.Replace(result, ":", "_", -1)
+	return strings.Trim(result, "_")
+}
+
+// DeleteFile delete file
+func DeleteFile(filename string) {
+	utils.DebugF("Delete: %v", filename)
+	os.Remove(utils.NormalizePath(filename))
+}
+
+// DeleteFolder delete entire folder
+func DeleteFolder(path string) {
+	utils.DebugF("Delete: %v", path)
+	os.RemoveAll(utils.NormalizePath(path))
+}
+
+// Append append content to a file
+func Append(dest string, src string) {
+	if !utils.FileExists(src) || utils.FileLength(src) <= 0 {
+		utils.DebugF("error to append %v", src)
+		return
+	}
+	data := utils.GetFileContent(src)
+	utils.AppendToContent(dest, data)
+}
+
+// Copy append content to a file
+func Copy(src string, dest string) {
+	if !utils.FileExists(src) || utils.FileLength(src) <= 0 {
+		return
+	}
+	input, _ := os.ReadFile(src)
+	os.WriteFile(dest, input, 0644)
+}
+
+// Sort sort content of a file
+func Sort(src string) {
+	data := utils.ReadingFileUnique(src)
+	if len(data) == 0 {
+		return
+	}
+	sort.Strings(data)
+	content := strings.Join(data, "\n")
+	// remove blank line
+	content = regexp.MustCompile(`[\t\r\n]+`).ReplaceAllString(strings.TrimSpace(content), "\n")
+	utils.WriteToFile(src, content)
+}
+
+// SortU sort content of a file
+func SortU(src string) {
+	if !utils.FileExists(src) {
+		utils.DebugF("File not found: %s", src)
+	}
+	cmd := fmt.Sprintf("LC_ALL=C sort -u -o %s %s", src, src)
+	utils.RunCmdWithOutput(cmd)
+}
+
+// Unique unique content of a file and remove blank line
+func Unique(filename string) {
+	data := utils.ReadingFileUnique(filename)
+	if len(data) > 0 {
+		content := strings.Join(data, "\n")
+		// remove blank line
+		content = regexp.MustCompile(`[\t\r\n]+`).ReplaceAllString(strings.TrimSpace(content), "\n")
+		utils.WriteToFile(filename, content)
+	}
+}
+
+// Compress sort content of a file
+func Compress(dest string, src string) {
+	if utils.FolderLength(src) < 1 {
+		utils.DebugF("Source folder is empty or not found: %s", src)
+		return
+	}
+	cmd := fmt.Sprintf("tar --use-compress-program='gzip -9' -C %s -cf %s .", strings.TrimRight(src, "/"), dest)
+	utils.RunCmdWithOutput(cmd)
+}
+
+// Decompress sort content of a file
+func Decompress(dest string, src string) {
+	if !utils.FileExists(src) {
+		utils.DebugF("File not found: %s", src)
+		return
+	}
+	utils.MakeDir(dest)
+	cmd := fmt.Sprintf("tar -xf %s -C %s", src, dest)
+	utils.RunCmdWithOutput(cmd)
+}
+
+func ExtractTarGz(filename string) error {
+	r, err := os.Open(utils.NormalizePath(filename))
+	if err != nil {
+		return err
+	}
+
+	uncompressedStream, err := gzip.NewReader(r)
+	if err != nil {
+		return err
+	}
+
+	tarReader := tar.NewReader(uncompressedStream)
+	for true {
+		header, err := tarReader.Next()
+		if err == io.EOF {
+			break
+		}
+
+		if err != nil {
+			return err
+		}
+
+		switch header.Typeflag {
+		case tar.TypeDir:
+			if err := os.Mkdir(header.Name, 0755); err != nil {
+				return err
+			}
+		case tar.TypeReg:
+			outFile, err := os.Create(header.Name)
+			if err != nil {
+				return err
+			}
+			if _, err := io.Copy(outFile, tarReader); err != nil {
+				return err
+			}
+			outFile.Close()
+
+		default:
+			return err
+
+		}
+
+	}
+
+	return nil
+}

execution/scripts_test.go

@@ -0,0 +1,19 @@
+package execution
+
+import (
+	"testing"
+)
+
+func TestChunkFile(t *testing.T) {
+	result := ChunkFileByPart("/tmp/oo/seqtest", 3)
+	t.Log(result)
+}
+
+func TestSort(t *testing.T) {
+	Sort("/tmp/sam")
+}
+
+func TestIsWildCard(t *testing.T) {
+	IsWildCard("github.com")
+	IsWildCard("tesla.com")
+}

execution/split.go

@@ -0,0 +1,126 @@
+package execution
+
+import (
+	"fmt"
+	"github.com/j3ssie/osmedeus/utils"
+	"github.com/robertkrimen/otto"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+)
+
+/*
+
+	SplitFile("src", "dest")
+ 	SplitFile("src", "base/dest", 500, "another/dest")
+ 	next step should be loop in file base/index
+
+*/
+
+// SplitFile split file into multiple file
+func SplitFile(kind string, arguments []otto.Value) {
+	source := arguments[0].String()
+	dest := arguments[1].String()
+	// prefix name of the output file
+	prefix := fmt.Sprintf("%v-chunked", filepath.Base(source))
+	destDir := filepath.Dir(utils.NormalizePath(dest))
+	if len(arguments) >= 4 {
+		destDir = arguments[3].String()
+	}
+	utils.MakeDir(destDir)
+
+	chunk := 200
+	// check if we change the size of it
+	if len(arguments) >= 3 {
+		chunkSize, _ := arguments[2].ToInteger()
+		chunk = int(chunkSize)
+	}
+
+	chunkParts := chunk
+	// get number of part
+	if kind == "size" {
+		length := utils.FileLength(source)
+		chunkParts = length / chunk
+	}
+
+	utils.DebugF("Splitting %v to %v %v", source, chunkParts, kind)
+	rawChunks, err := utils.SplitLineChunks(source, chunkParts)
+	if err != nil || len(rawChunks) == 0 {
+		utils.WarnF("error to split input file: %v", source)
+		return
+	}
+	fp, err := os.Open(source)
+	if err != nil {
+		utils.WarnF("error to open input file: %v", source)
+		return
+	}
+
+	var sumFile []string
+	for index, offset := range rawChunks {
+		targetName := path.Join(destDir, fmt.Sprintf("%v-%v", prefix, index))
+		reader := utils.NewRangeReader(fp, offset.Start, offset.Stop)
+		body := make([]byte, offset.Stop-offset.Start+1)
+		_, err := reader.Read(body)
+		if err != nil {
+			utils.WarnF("error to read chunk file: %s", err)
+			continue
+		}
+		sumFile = append(sumFile, targetName)
+		utils.DebugF("writing %v part to: %v", index, targetName)
+		utils.WriteToFile(targetName, string(body))
+	}
+
+	// write summary file
+	indexFile := path.Join(destDir, dest)
+	_, err = utils.WriteToFile(indexFile, strings.Join(sumFile, "\n"))
+	if err != nil {
+		utils.WarnF("Error writing to %v", indexFile)
+	}
+}
+
+// ChunkFileByPart chunk file to multiple part
+func ChunkFileByPart(source string, chunk int) [][]string {
+	var divided [][]string
+	data := utils.ReadingLines(source)
+	if len(data) <= 0 || chunk > len(data) {
+		if len(data) > 0 {
+			divided = append(divided, data)
+		}
+		return divided
+	}
+
+	chunkSize := (len(data) + chunk - 1) / chunk
+	for i := 0; i < len(data); i += chunkSize {
+		end := i + chunkSize
+		if end > len(data) {
+			end = len(data)
+		}
+
+		divided = append(divided, data[i:end])
+	}
+	return divided
+}
+
+// ChunkFileBySize chunk file to multiple part
+func ChunkFileBySize(source string, chunk int) [][]string {
+	var divided [][]string
+	data := utils.ReadingLines(source)
+	if len(data) <= 0 || chunk > len(data) {
+		if len(data) > 0 {
+			divided = append(divided, data)
+		}
+		return divided
+	}
+
+	chunkSize := chunk
+	for i := 0; i < len(data); i += chunkSize {
+		end := i + chunkSize
+		if end > len(data) {
+			end = len(data)
+		}
+
+		divided = append(divided, data[i:end])
+	}
+	return divided
+}

execution/wildcard.go

@@ -0,0 +1,61 @@
+package execution
+
+import (
+	"fmt"
+
+	//"github.com/OWASP/Amass/v3/requests"
+	//amassresolvers "github.com/OWASP/Amass/v3/resolvers"
+	"github.com/j3ssie/osmedeus/utils"
+)
+
+var defaultResolvers = []string{
+	"1.1.1.1:53",   // Cloudflare
+	"8.8.8.8:53",   // Google
+	"64.6.64.6:53", // Verisign
+	"8.8.4.4:53",   // Google Secondary
+}
+
+// IsWildCard check if target is wildcard or not
+func IsWildCard(domain string) bool {
+	//var resolvers []string
+	//resolvers = defaultResolvers
+	//resolverPool := amassresolvers.SetupResolverPool(resolvers, 1000, false, nil)
+	//if resolverPool == nil {
+	//	utils.WarnF("Failed to init DNS pool")
+	//	return false
+	//}
+	//
+	//ctx := context.Background()
+	//defer ctx.Done()
+	//subdomains := genSubs(domain, 5)
+	//var totalWildCard int
+	//for _, subdomain := range subdomains {
+	//	req := &requests.DNSRequest{
+	//		Name:   subdomain,
+	//		Domain: domain,
+	//	}
+	//	if !resolverPool.MatchesWildcard(ctx, req) {
+	//		utils.DebugF("[wild] %s\n", req.Name)
+	//	} else {
+	//		totalWildCard += 1
+	//		utils.DebugF("[wild] %s\n", req.Name)
+	//	}
+	//}
+	//utils.DebugF("Total number of wildcard: %v/%v\n", totalWildCard, len(subdomains))
+	//if totalWildCard == len(subdomains) {
+	//	utils.DebugF("Target %v is wildcard\n", domain)
+	//	return true
+	//}
+	return false
+}
+
+func genSubs(domain string, size int) []string {
+	var subdomains []string
+	subdomains = append(subdomains, fmt.Sprintf("notj3ssei.%s", domain))
+	subdomains = append(subdomains, fmt.Sprintf("verylong.%s", domain))
+
+	for i := 0; i < (size - 2); i++ {
+		subdomains = append(subdomains, fmt.Sprintf("%s.%s", utils.RandomString(5), domain))
+	}
+	return subdomains
+}

go.mod

@@ -0,0 +1,113 @@
+module github.com/j3ssie/osmedeus
+
+go 1.20
+
+require (
+	github.com/Jeffail/gabs/v2 v2.7.0
+	github.com/Shopify/yaml v2.1.0+incompatible
+	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751
+	github.com/aws/aws-sdk-go v1.44.327
+	github.com/cenkalti/backoff/v4 v4.2.1
+	github.com/davecgh/go-spew v1.1.1
+	github.com/dgrijalva/jwt-go v3.2.0+incompatible
+	github.com/digitalocean/godo v1.102.1
+	github.com/fatih/color v1.15.0
+	github.com/flosch/pongo2/v6 v6.0.0
+	github.com/fsnotify/fsnotify v1.6.0
+	github.com/go-playground/validator/v10 v10.15.1
+	github.com/go-resty/resty/v2 v2.7.0
+	github.com/go-telegram-bot-api/telegram-bot-api/v5 v5.5.1
+	github.com/gofiber/fiber/v2 v2.48.0
+	github.com/gofiber/jwt/v2 v2.2.7
+	github.com/gomarkdown/markdown v0.0.0-20230716120725-531d2d74bc12
+	github.com/hashicorp/go-version v1.6.0
+	github.com/jasonlvhit/gocron v0.0.1
+	github.com/jinzhu/copier v0.4.0
+	github.com/json-iterator/go v1.1.12
+	github.com/kyokomi/emoji v2.2.4+incompatible
+	github.com/linode/linodego v1.20.1
+	github.com/mackerelio/go-osstat v0.2.4
+	github.com/mitchellh/go-homedir v1.1.0
+	github.com/mitchellh/go-ps v1.0.0
+	github.com/olekukonko/tablewriter v0.0.5
+	github.com/panjf2000/ants v1.3.0
+	github.com/parnurzeal/gorequest v0.2.16
+	github.com/robertkrimen/otto v0.2.1
+	github.com/shirou/gopsutil v3.21.11+incompatible
+	github.com/sirupsen/logrus v1.9.3
+	github.com/slack-go/slack v0.12.2
+	github.com/spf13/cast v1.5.1
+	github.com/spf13/cobra v1.7.0
+	github.com/spf13/viper v1.16.0
+	github.com/swaggo/swag v1.16.1
+	github.com/thoas/go-funk v0.9.3
+	github.com/x-cray/logrus-prefixed-formatter v0.5.2
+	github.com/xanzy/go-gitlab v0.90.0
+	golang.org/x/crypto v0.12.0
+	golang.org/x/net v0.14.0
+	golang.org/x/oauth2 v0.11.0
+	golang.org/x/term v0.11.0
+	golang.org/x/text v0.12.0
+)
+
+require (
+	github.com/KyleBanks/depth v1.2.1 // indirect
+	github.com/PuerkitoBio/purell v1.1.1 // indirect
+	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
+	github.com/andybalholm/brotli v1.0.5 // indirect
+	github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a // indirect
+	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
+	github.com/go-openapi/jsonpointer v0.19.5 // indirect
+	github.com/go-openapi/jsonreference v0.19.6 // indirect
+	github.com/go-openapi/spec v0.20.4 // indirect
+	github.com/go-openapi/swag v0.19.15 // indirect
+	github.com/go-playground/locales v0.14.1 // indirect
+	github.com/go-playground/universal-translator v0.18.1 // indirect
+	github.com/golang-jwt/jwt/v4 v4.0.0 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/go-querystring v1.1.0 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/gorilla/websocket v1.4.2 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.4 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/klauspost/compress v1.16.3 // indirect
+	github.com/leodido/go-urn v1.2.4 // indirect
+	github.com/magiconair/properties v1.8.7 // indirect
+	github.com/mailru/easyjson v0.7.6 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/mattn/go-runewidth v0.0.14 // indirect
+	github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/pelletier/go-toml/v2 v2.0.8 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/rivo/uniseg v0.2.0 // indirect
+	github.com/smartystreets/goconvey v1.8.1 // indirect
+	github.com/spf13/afero v1.9.5 // indirect
+	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/subosito/gotenv v1.4.2 // indirect
+	github.com/tklauser/go-sysconf v0.3.12 // indirect
+	github.com/tklauser/numcpus v0.6.1 // indirect
+	github.com/valyala/bytebufferpool v1.0.0 // indirect
+	github.com/valyala/fasthttp v1.48.0 // indirect
+	github.com/valyala/tcplisten v1.0.0 // indirect
+	github.com/yusufpapurcu/wmi v1.2.3 // indirect
+	golang.org/x/sys v0.11.0 // indirect
+	golang.org/x/time v0.3.0 // indirect
+	golang.org/x/tools v0.7.0 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/protobuf v1.31.0 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
+	gopkg.in/sourcemap.v1 v1.0.5 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	moul.io/http2curl v1.0.0 // indirect
+)

go.sum

@@ -0,0 +1,737 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
+cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
+cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
+cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
+cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
+cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
+cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
+cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
+cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
+cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
+cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
+cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI=
+cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk=
+cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY=
+cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
+cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
+cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
+cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
+cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
+cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
+cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
+cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
+cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
+cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
+cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
+cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
+cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
+cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
+cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo=
+dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/Jeffail/gabs/v2 v2.7.0 h1:Y2edYaTcE8ZpRsR2AtmPu5xQdFDIthFG0jYhu5PY8kg=
+github.com/Jeffail/gabs/v2 v2.7.0/go.mod h1:dp5ocw1FvBBQYssgHsG7I1WYsiLRtkUaB1FEtSwvNUw=
+github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc=
+github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE=
+github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
+github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
+github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
+github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
+github.com/Shopify/yaml v2.1.0+incompatible h1:Y7Wj6eQo5Byeaxb2M9FiPdKBQ3ooLsRIEmUZMMxCI8g=
+github.com/Shopify/yaml v2.1.0+incompatible/go.mod h1:Mrsv1G5Osez+VdHYcSI2zfaUzPu4lCdO9cW8R2lHPEc=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/andybalholm/brotli v1.0.2/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
+github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
+github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
+github.com/aws/aws-sdk-go v1.44.327 h1:ZS8oO4+7MOBLhkdwIhgtVeDzCeWOlTfKJS7EgggbIEY=
+github.com/aws/aws-sdk-go v1.44.327/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
+github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
+github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/digitalocean/godo v1.102.1 h1:BrNePwIXjQWjOJXVTBqkURMjm70BRR0qXbRKfHNBF24=
+github.com/digitalocean/godo v1.102.1/go.mod h1:SaUYccN7r+CO1QtsbXGypAsgobDrmSfVMJESEfXgoEg=
+github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU=
+github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
+github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
+github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
+github.com/flosch/pongo2/v6 v6.0.0 h1:lsGru8IAzHgIAw6H2m4PCyleO58I40ow6apih0WprMU=
+github.com/flosch/pongo2/v6 v6.0.0/go.mod h1:CuDpFm47R0uGGE7z13/tTlt1Y6zdxvr2RLT5LJhsHEU=
+github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
+github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
+github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
+github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
+github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
+github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
+github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
+github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
+github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
+github.com/go-openapi/jsonreference v0.19.6 h1:UBIxjkht+AWIgYzCDSv2GN+E/togfwXUJFRTWhl2Jjs=
+github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns=
+github.com/go-openapi/spec v0.20.4 h1:O8hJrt0UMnhHcluhIdUgCLRWyM2x7QkBXRvOs7m+O1M=
+github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I=
+github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
+github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM=
+github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
+github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
+github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
+github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
+github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
+github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
+github.com/go-playground/validator/v10 v10.15.1 h1:BSe8uhN+xQ4r5guV/ywQI4gO59C2raYcGffYWZEjZzM=
+github.com/go-playground/validator/v10 v10.15.1/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
+github.com/go-redis/redis v6.15.5+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
+github.com/go-resty/resty/v2 v2.7.0 h1:me+K9p3uhSmXtrBZ4k9jcEAfJmuC8IivWHwaLZwPrFY=
+github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I=
+github.com/go-telegram-bot-api/telegram-bot-api/v5 v5.5.1 h1:wG8n/XJQ07TmjbITcGiUaOtXxdrINDz1b0J1w0SzqDc=
+github.com/go-telegram-bot-api/telegram-bot-api/v5 v5.5.1/go.mod h1:A2S0CWkNylc2phvKXWBBdD3K0iGnDBGbzRpISP2zBl8=
+github.com/go-test/deep v1.0.4 h1:u2CU3YKy9I2pmu9pX0eq50wCgjfGIt539SqR7FbHiho=
+github.com/go-test/deep v1.0.4/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
+github.com/gofiber/fiber/v2 v2.17.0/go.mod h1:iftruuHGkRYGEXVISmdD7HTYWyfS2Bh+Dkfq4n/1Owg=
+github.com/gofiber/fiber/v2 v2.48.0 h1:cRVMCb9aUJDsyHxGFLwz/sGzDggdailZZyptU9F9cU0=
+github.com/gofiber/fiber/v2 v2.48.0/go.mod h1:xqJgfqrc23FJuqGOW6DVgi3HyZEm2Mn9pRqUb2kHSX8=
+github.com/gofiber/jwt/v2 v2.2.7 h1:MgXZV+ak+FiRVepD3btHBxWcyxlFzTDGXJv78dU1sIE=
+github.com/gofiber/jwt/v2 v2.2.7/go.mod h1:yaOHLccYXJidk1HX/EiIdIL+Z1xmY2wnIv6hgViw384=
+github.com/golang-jwt/jwt/v4 v4.0.0 h1:RAqyYixv1p7uEnocuy8P1nru5wprCh/MH2BIlW5z5/o=
+github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
+github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gomarkdown/markdown v0.0.0-20230716120725-531d2d74bc12 h1:uK3X/2mt4tbSGoHvbLBHUny7CKiuwUip3MArtukol4E=
+github.com/gomarkdown/markdown v0.0.0-20230716120725-531d2d74bc12/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
+github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
+github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g=
+github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g=
+github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
+github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
+github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
+github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
+github.com/hashicorp/go-hclog v1.2.0 h1:La19f8d7WIlm4ogzNHB0JGqs5AUDAZ2UfCY4sJXcJdM=
+github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA=
+github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
+github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek=
+github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
+github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/jasonlvhit/gocron v0.0.1 h1:qTt5qF3b3srDjeOIR4Le1LfeyvoYzJlYpqvG7tJX5YU=
+github.com/jasonlvhit/gocron v0.0.1/go.mod h1:k9a3TV8VcU73XZxfVHCHWMWF9SOqgoku0/QlY2yvlA4=
+github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8=
+github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
+github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
+github.com/klauspost/compress v1.16.3 h1:XuJt9zzcnaz6a16/OU53ZjWp/v7/42WcR5t2a0PcNQY=
+github.com/klauspost/compress v1.16.3/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kyokomi/emoji v2.2.4+incompatible h1:np0woGKwx9LiHAQmwZx79Oc0rHpNw3o+3evou4BEPv4=
+github.com/kyokomi/emoji v2.2.4+incompatible/go.mod h1:mZ6aGCD7yk8j6QY6KICwnZ2pxoszVseX1DNoGtU2tBA=
+github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
+github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
+github.com/linode/linodego v1.20.1 h1:IW3SrZjRzrclYZnzFd80f8lSkTYAM7gVTJW0t7HnFKQ=
+github.com/linode/linodego v1.20.1/go.mod h1:ggoWnJXssx9wPWNnR3x7WaOpOBOEhsPB/HO7iflF5qY=
+github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs=
+github.com/mackerelio/go-osstat v0.2.4/go.mod h1:Zy+qzGdZs3A9cuIqmgbJvwbmLQH9dJvtio5ZjJTbdlQ=
+github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
+github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
+github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA=
+github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
+github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
+github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI=
+github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc=
+github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg=
+github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
+github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
+github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo=
+github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
+github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/panjf2000/ants v1.3.0 h1:8pQ+8leaLc9lys2viEEr8md0U4RN6uOSUCE9bOYjQ9M=
+github.com/panjf2000/ants v1.3.0/go.mod h1:AaACblRPzq35m1g3enqYcxspbbiOJJYaxU2wMpm1cXY=
+github.com/parnurzeal/gorequest v0.2.16 h1:T/5x+/4BT+nj+3eSknXmCTnEVGSzFzPGdpqmUVVZXHQ=
+github.com/parnurzeal/gorequest v0.2.16/go.mod h1:3Kh2QUMJoqw3icWAecsyzkpY7UzRfDhbRdTjtNwNiUE=
+github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ=
+github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/robertkrimen/otto v0.2.1 h1:FVP0PJ0AHIjC+N4pKCG9yCDz6LHNPCwi/GKID5pGGF0=
+github.com/robertkrimen/otto v0.2.1/go.mod h1:UPwtJ1Xu7JrLcZjNWN8orJaM5n5YEtqL//farB5FlRY=
+github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI=
+github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
+github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
+github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/slack-go/slack v0.12.2 h1:x3OppyMyGIbbiyFhsBmpf9pwkUzMhthJMRNmNlA4LaQ=
+github.com/slack-go/slack v0.12.2/go.mod h1:hlGi5oXA+Gt+yWTPP0plCdRKmjsDxecdHxYQdlMQKOw=
+github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGByCY=
+github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY=
+github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60=
+github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM=
+github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ=
+github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA=
+github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48=
+github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
+github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
+github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
+github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v1.16.0 h1:rGGH0XDZhdUOryiDWjmIvUSWpbNqisK8Wk0Vyefw8hc=
+github.com/spf13/viper v1.16.0/go.mod h1:yg78JgCJcbrQOvV9YLXgkLaZqUidkY9K+Dd1FofRzQg=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY=
+github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8=
+github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0=
+github.com/swaggo/swag v1.16.1 h1:fTNRhKstPKxcnoKsytm4sahr8FaYzUcT7i1/3nd/fBg=
+github.com/swaggo/swag v1.16.1/go.mod h1:9/LMvHycG3NFHfR6LwvikHv5iFvmPADQ359cKikGxto=
+github.com/thoas/go-funk v0.9.3 h1:7+nAEx3kn5ZJcnDm2Bh23N2yOtweO14bi//dvRtgLpw=
+github.com/thoas/go-funk v0.9.3/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q=
+github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU=
+github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
+github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk=
+github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY=
+github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasthttp v1.26.0/go.mod h1:cmWIqlu99AO/RKcp1HWaViTqc57FswJOfYYdPJBl8BA=
+github.com/valyala/fasthttp v1.48.0 h1:oJWvHb9BIZToTQS3MuQ2R3bJZiNSa2KiNdeI8A+79Tc=
+github.com/valyala/fasthttp v1.48.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA=
+github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
+github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
+github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJzfthRT6usrui8uGmg=
+github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE=
+github.com/xanzy/go-gitlab v0.90.0 h1:j8ZUHfLfXdnC+B8njeNaW/kM44c1zw8fiuNj7D+qQN8=
+github.com/xanzy/go-gitlab v0.90.0/go.mod h1:5ryv+MnpZStBH8I/77HuQBsMbBGANtVpLWC15qOjWAw=
+github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw=
+github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
+golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk=
+golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
+golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
+golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
+golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
+golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
+golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM=
+golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
+golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14=
+golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU=
+golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
+golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0=
+golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU=
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc=
+golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
+golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
+golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
+golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4=
+golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
+google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
+google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
+google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
+google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE=
+google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
+google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
+google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
+google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
+google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
+google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
+google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=
+google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
+google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
+gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/sourcemap.v1 v1.0.5 h1:inv58fC9f9J3TK2Y2R1NPntXEn3/wjWHkonhIUODNTI=
+gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+moul.io/http2curl v1.0.0 h1:6XwpyZOYsgZJrU8exnG87ncVkU1FVCcTRpwzOkTDUi8=
+moul.io/http2curl v1.0.0/go.mod h1:f6cULg+e4Md/oW1cYmwW4IWQOVl2lGbmCNGOHvzX2kE=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
+rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

libs/cloud.go

@@ -0,0 +1,115 @@
+package libs
+
+// Cloud struct define folder to push data
+type Cloud struct {
+	CheckingLimit      bool
+	ReBuildBaseImage   bool
+	IgnoreConfigFile   bool
+	BackgroundRun      bool
+	OnlyCreateDroplet  bool
+	OnlyCreateInstance bool
+	EnablePrivateIP    bool
+
+	EnableSyncWorkflow   bool
+	AddNewProvider       bool
+	RemoteWorkflowFolder string
+	TokensFile           string
+
+	CopyWorkspaceToGit bool
+	ClearTime          string
+	InstanceName       string
+	TempTarget         string
+	// content of secret key to avoid reading it too much
+	SecretKeyContent string
+	PublicKeyContent string
+
+	// enable terraform
+	EnableTerraform bool
+
+	// chunk options
+	ChunkInputs      string
+	BaseWorkspace    string
+	LocalSyncFolder  string
+	DisableLocalSync bool
+	RemoteRunList    bool
+	TargetAsFile     bool
+	EnableChunk      bool
+	IgnoreProcess    bool
+	NumberOfParts    int
+	Threads          int
+
+	// specific cloud instance resources
+	Size        string
+	Region      string
+	Token       string
+	Provider    string
+	IgnoreSetup bool
+
+	// for pre-commands and post-commands
+	RemotePreRun []string
+	// run script on local machine after scan done
+	LocalSteps   []Step `yaml:"local_steps"`
+	LocalPreRun  []string
+	LocalPostRun []string
+
+	// use to clone build-osm repo
+	SecretKey   string
+	PublicKey   string
+	BuildRepo   string
+	Binary      string
+	Retry       int
+	UnzipResult bool
+
+	// for health check
+	ForEverHealthCheck bool
+	NoDelete           bool
+
+	// raw command here
+	Extra      string
+	Flow       string
+	Module     string
+	Workspace  string
+	RawCommand string
+	Params     []string
+
+	WsSource string
+	WsDest   string
+
+	Input      string
+	Inputs     []string
+	InputsFile string
+
+	Target map[string]string
+}
+
+// Request all information about request
+type Request struct {
+	Timeout  int
+	Repeat   int
+	Scheme   string
+	Host     string
+	Port     string
+	Path     string
+	URL      string
+	Proxy    string
+	Method   string
+	Redirect bool
+	Headers  []map[string]string
+	Body     string
+	Beautify string
+}
+
+// Response all information about response
+type Response struct {
+	HasPopUp       bool
+	StatusCode     int
+	Status         string
+	ContentType    string
+	Headers        []map[string]string
+	Body           string
+	ResponseTime   float64
+	Length         int
+	Beautify       string
+	Location       string
+	BeautifyHeader string
+}

libs/flow.go

@@ -0,0 +1,69 @@
+package libs
+
+// Routine for each scan
+type Routine struct {
+	RoutineName   string
+	FlowFolder    string `yaml:"flow"`
+	Timeout       string `yaml:"timeout"`
+	ParsedModules []Module
+	Modules       []string
+}
+
+// Flow struct to define specific field for a mode
+type Flow struct {
+	NoDB        bool `yaml:"nodb"`
+	SkipIndexed bool `yaml:"skip-indexed"`
+	ForceParams bool `yaml:"force-params"`
+	Input       string
+	Validator   string // domain, cidr, ip or domain-file, cidr-file and so on
+
+	Name        string
+	Type        string
+	DefaultType string
+	Desc        string
+	Usage       string
+
+	Params   []map[string]string
+	Routines []Routine
+
+	RemotePreRun []string `yaml:"remote_pre_run"`
+	// run script on local machine after scan done
+	LocalPreRun  []string `yaml:"local_pre_run"`
+	LocalPostRun []string `yaml:"local_post_run"`
+}
+
+// Module struct to define specific field for a module
+type Module struct {
+	NoDB        bool   `yaml:"nodb"`
+	Validator   string // domain, cidr, ip
+	ForceParams bool   `yaml:"force-params"`
+
+	// just for print some info
+	Name  string
+	Desc  string
+	Usage string
+
+	// enable resume, if all reports file exist then skip the module
+	Resume bool
+	// run module despite resume enable
+	Forced bool
+
+	MTimeout   string `yaml:"mtimeout"`
+	Params     []map[string]string
+	ModulePath string
+
+	PreRun []string `yaml:"pre_run"`
+	Report struct {
+		Final []string
+		Noti  []string
+		Diff  []string
+	}
+	Steps   []Step
+	PostRun []string `yaml:"post_run"`
+
+	RemotePreRun []string `yaml:"remote_pre_run"`
+	// run script on local machine after scan done
+	LocalSteps   []Step   `yaml:"local_steps"`
+	LocalPreRun  []string `yaml:"local_pre_run"`
+	LocalPostRun []string `yaml:"local_post_run"`
+}

libs/mics.go

@@ -0,0 +1,61 @@
+package libs
+
+// Cdn credentials for other client
+type Cdn struct {
+	Bucket      string
+	Region      string
+	SecretKey   string
+	AccessKeyId string
+}
+
+// Git credentials for other client
+type Git struct {
+	BaseURL       string
+	Username      string
+	Password      string
+	Token         string
+	Group         string
+	DefaultPrefix string
+	DefaultTag    string
+	DefaultUser   string
+	DefaultUID    int
+	DeStorage     string
+}
+
+// TmuxOpt credentials for other client
+type TmuxOpt struct {
+	ApplyAll       bool
+	SelectedWindow string
+	Exclude        string
+	Limit          int
+}
+
+// Cron credentials for other client
+type Cron struct {
+	Command  string
+	Schedule int
+	Forever  bool
+}
+
+// Remote credentials for other client
+type Remote struct {
+	MasterHost string
+	MasterCred string
+	PoolHost   string
+	PoolCred   string
+}
+
+// Sync credentials for other client
+type Sync struct {
+	BaseURL string
+	Prefix  string
+	Pool    string
+}
+
+// Client credentials for other client
+type Client struct {
+	Username string
+	Password string
+	JWT      string
+	URL      string
+}

libs/noti.go

@@ -0,0 +1,26 @@
+package libs
+
+// Notification struct define notification method
+type Notification struct {
+	ClientName string
+	// SlacksWebHooks list
+	SlacksWebHooks map[string]string
+	// TelegramWebHooks list
+	TelegramWebHooks map[string]string
+	// Telegram part
+	TelegramToken            string
+	TelegramChannel          string
+	TelegramStatusChannel    string
+	TelegramReportChannel    string
+	TelegramDirbChannel      string
+	TelegramSensitiveChannel string
+	TelegramMicsChannel      string
+	// use this when we want to send a file to channel
+	SlackWebHook       string
+	SlackToken         string
+	SlackReportChannel string
+	SlackStatusChannel string
+	SlackDiffChannel   string
+	// later then
+	DiscordToken string
+}

libs/options.go

@@ -0,0 +1,172 @@
+package libs
+
+// Options global options
+type Options struct {
+	ConfigFile  string
+	LogFile     string
+	Concurrency int
+
+	// default threads hold for running module
+	Tactics string
+	Threads int
+
+	Timeout           string
+	EnableFormatInput bool
+	Verbose           bool
+	FullHelp          bool
+
+	// some disable options
+	NoPostRun            bool
+	NoPreRun             bool
+	NoNoti               bool
+	NoBanner             bool
+	NoGit                bool
+	NoClean              bool
+	NoDB                 bool
+	NoCdn                bool
+	DisableValidateInput bool
+
+	PremiumPackage   bool
+	Resume           bool
+	Quite            bool
+	Force            bool
+	WildCardCheck    bool
+	Debug            bool
+	EnableDeStorage  bool
+	PID              int
+	SyncTimes        int
+	PollingTime      int
+	MDCodeBlockLimit int
+	Exclude          []string
+	Params           []string
+	CustomGit        bool
+	EnableBackup     bool
+	JsonOutput       bool
+
+	Client Client
+	Queue  Queue
+	Git    Git
+	Sync   Sync
+	Scan   Scan
+	Server Server
+	Env    Environment
+	Noti   Notification
+	Flow   Flow
+	Module Module
+	Tmux   TmuxOpt
+	Cron   Cron
+	Remote Remote
+	Cdn    Cdn
+	Update Update
+
+	ThreadsHold     ThreadsHold
+	Cloud           Cloud
+	Report          Report
+	CloudConfigFile string
+	TokenConfigFile string
+	GitSync         bool
+
+	ScanID   string
+	Storages map[string]string
+}
+
+// Scan sub options for scan
+type Scan struct {
+	ROptions   map[string]string
+	Params     []string
+	Input      string
+	InputType  string // domain, url, ip, cidr or domainList, urlList, ipList, cidrList
+	ParamsFile string
+	Inputs     []string
+	InputList  string
+	Modules    []string
+	Flow       string
+
+	BaseWorkspace   string
+	CustomWorkspace string
+	SuffixName      string
+	Force           bool
+	// this is true when calling from cloud scan
+	RemoteCall bool
+}
+
+type ThreadsHold struct {
+	Default    int
+	Aggressive int
+	Gently     int
+}
+
+// Report sub options for report
+type Report struct {
+	CustomPreFix  string
+	PublicIP      string
+	ExtractFolder string
+	Static        bool
+	Raw           bool
+}
+
+// Server sub options for api server
+type Server struct {
+	DisableWorkspaceListing bool
+	DisableSSL              bool
+	PreFork                 bool
+	NoAuthen                bool
+
+	PollingTime    int
+	Bind           string
+	Port           string
+	StaticPrefix   string
+	JWTSecret      string
+	Cors           string
+	UIPath         string
+	MasterPassword string
+
+	// database
+	DBPath       string
+	DBType       string
+	DBConnection string
+	DBName       string
+	DBUser       string
+	DBPass       string
+	DBHost       string
+	DBPort       string
+
+	// for SSL
+	CertFile string
+	KeyFile  string
+}
+
+// Storage struct define folder to push data
+type Storage struct {
+	SecretKey      string
+	SummaryStorage string
+	SummaryRepo    string
+	HTTPStorage    string
+	HTTPRepo       string
+	AssetsStorage  string
+	AssetsRepo     string
+}
+
+// Environment some config path
+type Environment struct {
+	RootFolder       string // ~/.osmedeus
+	StoragesFolder   string // ~/.osmedeus/storages/
+	WorkspacesFolder string // ~/workspaces-osmedeus/
+
+	// Base one
+	BaseFolder      string // ~/osmedeus-base
+	BinariesFolder  string // ~/osmedeus-base/binaries
+	DataFolder      string // ~/osmedeus-base/data/
+	OseFolder       string // ~/osmedeus-base/ose/
+	WorkFlowsFolder string // ~/osmedeus-base/workflow/
+
+	// cloud stuff
+	CloudConfigFolder string // ~/osmedeus-base/clouds/
+	ProviderFolder    string // ~/.osmedeus/providers/
+	InstancesFolder   string // ~/.osmedeus/instances/
+	BackupFolder      string
+
+	// Mics
+	ScriptsFolder string
+	UIFolder      string
+}

libs/queue.go

@@ -0,0 +1,22 @@
+package libs
+
+// Queue sub options for quque
+type Queue struct {
+	QueueFolder string
+	QueueFile   string
+	RawCommand  string
+
+	InputAsFile bool
+	Add         bool
+}
+
+type InputFormat struct {
+	Input       string   `json:"input"`
+	Flow        string   `json:"flow"`
+	Modules     []string `json:"module"`
+	Params      []string `json:"params"`
+	Workspaces  string   `json:"workspace"`
+	Extra       string   `json:"extra"`
+	Command     string   `json:"command"`
+	InputAsFile bool     `json:"input-as-file"`
+}

libs/step.go

@@ -0,0 +1,30 @@
+package libs
+
+// Step struct to define component about a command
+type Step struct {
+	// timeout for commands and script
+	Timeout string
+	// use for run loop command
+	Parallel int
+	Threads  string
+	Source   string
+
+	Label string
+
+	Conditions []string
+	Required   []string
+
+	Commands []string
+	Ose      []string `yaml:"ose"`
+	Scripts  []string
+
+	// run when conditions are false
+	RCommands []string `yaml:"rcommands"`
+	RScripts  []string `yaml:"rscripts"`
+
+	// post condition and script
+	PConditions []string
+	PScripts    []string
+
+	Std string
+}

libs/update.go

@@ -0,0 +1,28 @@
+package libs
+
+// Update some config path
+type Update struct {
+	UpdateURL    string // url to download the update script
+	UpdateScript string
+	MetaDataURL  string
+	UpdateKey    string //
+	UpdateType   string // git, http
+	UpdateConfig string // ~/.osmedeus/update
+
+	UpdateVersion string
+	UpdateFolder  string
+	UpdateDate    string
+	CleanOldData  bool
+	VulnUpdate    bool
+	GenerateMeta  string
+	ForceUpdate   bool
+	IsUpdateBin   bool
+	EnableUpdate  bool
+	NoUpdate      bool
+}
+
+type UpdateMetaData struct {
+	WorkflowVersion string `json:"workflow_version"`
+	CoreVersion     string `json:"core_version"`
+	UpdatedAt       string `json:"updated_at"`
+}

libs/version.go

@@ -0,0 +1,26 @@
+package libs
+
+import "fmt"
+
+const (
+	// VERSION of this project
+	VERSION = "v4.6.0"
+	// DESC description of the tool
+	DESC = "A Workflow Engine for Offensive Security"
+	// BINARY name of osmedeus
+	BINARY = "osmedeus"
+	// SNAPSHOT binary name of osmedeus
+	SNAPSHOT = "osm"
+	// AUTHOR of this
+	AUTHOR = "@j3ssiejjj"
+	// DOCS private document
+	DOCS = "https://docs.osmedeus.org"
+	// METADATA domain for checking update
+	METADATA = "https://metadata.osmedeus.org"
+	// INSTALL default install script
+	INSTALL = "https://raw.githubusercontent.com/osmedeus/osmedeus-base/master/install.sh"
+)
+
+// TEMP default folder to store inputs
+var TEMP = fmt.Sprintf("/tmp/%s-inputs/", SNAPSHOT)
+var LDIR = fmt.Sprintf("/tmp/%s-log/", SNAPSHOT)

main.go

@@ -0,0 +1,7 @@
+package main
+
+import "github.com/j3ssie/osmedeus/cmd"
+
+func main() {
+	cmd.Execute()
+}

provider/action.go

@@ -0,0 +1,201 @@
+package provider
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/cenkalti/backoff/v4"
+	"github.com/fatih/color"
+	"github.com/j3ssie/osmedeus/utils"
+	"github.com/spf13/cast"
+)
+
+const (
+	GetSSHKey       = "get-sshkey"
+	RunBuild        = "run-build"
+	ListImage       = "list-image"
+	ListInstance    = "list-instance"
+	GetInstanceInfo = "get-instance"
+	CreateInstance  = "create-instance"
+	BootInstance    = "boot-instance"
+)
+
+func (p *Provider) GetSSHKey() (err error) {
+	if strings.TrimSpace(p.SSHPublicKey) == "" {
+		return fmt.Errorf("error getting SSHKey -- blank ssh public key")
+	}
+
+	switch p.ProviderName {
+	case "do", "digitalocean":
+		err = p.GetSSHKeyDO()
+	case "ln", "line", "linode":
+		err = p.GetSSHKeyLN()
+	case "aw", "aws", "asw":
+		err = p.GetSSHKeyAWS()
+	default:
+		err = p.GetSSHKeyDO()
+	}
+	return err
+}
+
+func (p *Provider) ListInstance() (err error) {
+	switch p.ProviderName {
+	case "do", "digitalocean":
+		err = p.ListInstanceDO()
+	case "ln", "line", "linode":
+		err = p.ListInstanceLN()
+	case "aw", "aws", "asw":
+		err = p.ListInstanceAWS()
+	default:
+		err = p.ListInstanceDO()
+	}
+	return err
+}
+
+func (p *Provider) ListSnapShot() (err error) {
+	switch p.ProviderName {
+	case "do", "digitalocean":
+		err = p.ListSnapshotDO()
+	case "ln", "line", "linode":
+		err = p.ListSnapshotLN()
+	case "aw", "aws", "asw":
+		err = p.ListSnapshotAWS()
+	default:
+		err = p.ListSnapshotDO()
+	}
+
+	if !p.IsBackgroundCheck {
+		utils.InforF("Found base image snapshot with ID: %v", color.HiCyanString(p.SnapshotID))
+	}
+	return err
+}
+
+func (p *Provider) CreateInstance(name string) (err error) {
+	var id interface{}
+	operation := func() error {
+		switch p.ProviderName {
+		case "do", "digitalocean":
+			id, err = p.CreateInstanceDO(name)
+			if err == nil {
+				err = p.Action(GetInstanceInfo, id)
+			}
+		case "ln", "line", "linode":
+			id, err = p.CreateInstanceLN(name)
+			if err == nil {
+				err = p.Action(BootInstance, id)
+				err = p.Action(GetInstanceInfo, id)
+			}
+
+		case "aw", "aws", "asw":
+			id, err = p.CreateInstanceAWS(name)
+			if err == nil {
+				err = p.Action(BootInstance, id)
+				err = p.Action(GetInstanceInfo, id)
+			}
+		default:
+			id, err = p.CreateInstanceDO(name)
+			if err == nil {
+				err = p.Action(GetInstanceInfo, id)
+			}
+		}
+		return err
+	}
+	err = backoff.Retry(operation, p.BackOff)
+	if err != nil {
+		utils.WarnF("error create instance action %v -- %v", p.ProviderName, name)
+		return err
+	}
+	return nil
+}
+
+// func (p *Provider) CreateInstanceF(name string) (err error) {
+// 	var id int
+// 	switch p.ProviderName {
+// 	case "do", "digitalocean":
+// 		id, err = p.CreateInstanceDO(name)
+// 		if err == nil {
+// 			err = p.Action(GetInstanceInfo, id)
+// 		}
+// 	case "ln", "line", "linode":
+// 		id, err = p.CreateInstanceLN(name)
+// 		if err == nil {
+// 			err = p.Action(BootInstance, id)
+// 			err = p.Action(GetInstanceInfo, id)
+// 		}
+// 	default:
+// 		id, err = p.CreateInstanceDO(name)
+// 		if err == nil {
+// 			err = p.Action(GetInstanceInfo, id)
+// 		}
+// 	}
+// 	return err
+// }
+
+func (p *Provider) BootInstance(id interface{}) (err error) {
+	switch p.ProviderName {
+	case "do", "digitalocean":
+	case "ln", "line", "linode":
+		err = p.BootInstanceLN(cast.ToInt(id))
+	case "aw", "aws", "asw":
+		// err = p.AllowRootAccessAWS(cast.ToString(id))
+	default:
+		err = p.BootInstanceLN(cast.ToInt(id))
+	}
+	if err != nil {
+		utils.WarnF("error booting instance: %v", id)
+		return err
+	}
+	return nil
+}
+
+func (p *Provider) GetInstanceInfo(id interface{}) (err error) {
+	var instance Instance
+	switch p.ProviderName {
+	case "do", "digitalocean":
+		instance, err = p.InstanceInfoDO(cast.ToInt(id))
+	case "ln", "line", "linode":
+		instance, err = p.InstanceInfoLN(cast.ToInt(id))
+	case "aw", "aws", "asw":
+		instance, err = p.InstanceInfoAWS(cast.ToString(id))
+	default:
+		instance, err = p.InstanceInfoDO(cast.ToInt(id))
+	}
+	if err != nil {
+		utils.WarnF("error getting public IP of instance: %v", color.HiBlueString("%v", id))
+		return err
+	}
+	p.Instances = append(p.Instances, instance)
+	return nil
+}
+
+func (p *Provider) DeleteInstance(id string) (err error) {
+	utils.DebugF("[%v] Delete instance: %v", p.ProviderName, id)
+	switch p.ProviderName {
+	case "do", "digitalocean":
+		err = p.DeleteInstanceDO(id)
+	case "ln", "line", "linode":
+		err = p.DeleteInstanceLN(id)
+	case "aw", "aws", "asw":
+		err = p.DeleteInstanceAWS(id)
+	default:
+		err = p.DeleteInstanceDO(id)
+	}
+	return err
+}
+
+func (p *Provider) DeleteOldSnapshot() (err error) {
+	for _, id := range p.OldSnapShotID {
+		switch p.ProviderName {
+		case "do", "digitalocean":
+			err = p.DeleteSnapShotDO(id)
+		case "ln", "line", "linode":
+			err = p.DeleteSnapShotLN(id)
+		case "aw", "aws", "asw":
+			err = p.DeleteImageAWS(id)
+		default:
+			err = p.DeleteSnapShotDO(id)
+		}
+	}
+
+	return err
+}

provider/building.go

@@ -0,0 +1,157 @@
+package provider
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/fatih/color"
+	"github.com/j3ssie/osmedeus/libs"
+	"github.com/j3ssie/osmedeus/utils"
+)
+
+func (p *Provider) PrePareBuildData() {
+	contentFile := path.Join(p.Opt.Env.CloudConfigFolder, fmt.Sprintf("providers/%s.provider", p.ProviderName))
+	content := utils.GetFileContent(contentFile)
+
+	data := make(map[string]string)
+	data["snapshot_name"] = p.SnapshotName
+	data["api_token"] = p.Token
+	// for aws only
+	data["access_key"] = p.AccessKeyId
+	data["secret_key"] = p.SecretKey
+	data["source_ami"] = p.ProviderConfig.DefaultImage
+
+	// c.Cloud.ProviderFolder --> ~/.osmedeus/provider/<osmp-name>-v4.x-randomstring
+	p.ProviderConfig.ProviderFolder = path.Join(p.Opt.Env.ProviderFolder, fmt.Sprintf("%s-%s", p.SnapshotName, utils.RandomString(6)))
+	utils.MakeDir(p.ProviderConfig.ProviderFolder)
+	data["ProviderFolder"] = p.ProviderConfig.ProviderFolder
+
+	data["image"] = p.ProviderConfig.DefaultImage
+	data["size"] = p.ProviderConfig.Size
+	data["region"] = p.ProviderConfig.Region
+	data["TS"] = utils.GetTS()
+
+	// generate packer content file to run
+	providerString := utils.RenderText(content, data)
+	data["Builder"] = providerString
+
+	// ~/osmedeus-base
+	data["BaseFolder"] = utils.NormalizePath(strings.TrimLeft(p.Opt.Env.BaseFolder, "/"))
+	data["Plugins"] = p.Opt.Env.BinariesFolder
+	data["OBin"] = p.Opt.Env.BinariesFolder
+	data["Data"] = p.Opt.Env.DataFolder
+	data["Cloud"] = p.Opt.Env.CloudConfigFolder
+	data["Workflow"] = p.Opt.Env.WorkFlowsFolder
+
+	// ~/.osmedeus/workspaces
+	data["Workspaces"] = p.Opt.Env.WorkspacesFolder
+	data["Binary"] = libs.BINARY
+	data["VERSION"] = libs.VERSION
+	data["BuildRepo"] = p.Opt.Cloud.BuildRepo
+
+	// for terraform
+	data["ssh_public_key"] = p.Opt.Cloud.PublicKeyContent
+	data["root_password"] = fmt.Sprintf("%s-%s", libs.SNAPSHOT, utils.RandomString(8))
+
+	//spew.Dump("data --> ", data)
+	//spew.Dump("p.ProviderConfig --> ", p.ProviderConfig)
+
+	p.ProviderConfig.BuildData = data
+}
+
+func (p *Provider) BuildImage() (err error) {
+	if p.SnapshotFound && !p.Opt.Cloud.ReBuildBaseImage {
+		return nil
+	}
+
+	p.PrePareBuildData()
+	p.DeleteOldSnapshot()
+
+	// p.ProviderConfig.ProviderFolder --> ~/.osmedeus/provider/<osmp-name>
+
+	utils.DebugF("Cleaning old provider build: %s", p.ProviderConfig.ProviderFolder)
+	os.RemoveAll(p.ProviderConfig.ProviderFolder)
+	utils.MakeDir(p.ProviderConfig.ProviderFolder)
+
+	// generate provision process
+	setupContent := utils.GetFileContent(path.Join(p.Opt.Env.CloudConfigFolder, "setup.sh"))
+	setupContent = utils.RenderText(setupContent, p.ProviderConfig.BuildData)
+	setupFile := path.Join(p.ProviderConfig.ProviderFolder, "setup.sh")
+	utils.WriteToFile(setupFile, setupContent)
+
+	// generate build file
+	var buildContent string
+	buildContentFile := path.Join(p.Opt.Env.CloudConfigFolder, "general-build.packer")
+	switch p.ProviderName {
+	case "do", "digitalocean":
+		buildContentFile = path.Join(p.Opt.Env.CloudConfigFolder, "digitalocean-build.packer")
+		if !utils.FileExists(buildContentFile) {
+			buildContentFile = path.Join(p.Opt.Env.CloudConfigFolder, "do-build.packer")
+			if !utils.FileExists(buildContentFile) {
+				buildContentFile = path.Join(p.Opt.Env.CloudConfigFolder, "general-build.packer")
+			}
+		}
+
+	case "ln", "line", "linode":
+		buildContentFile = path.Join(p.Opt.Env.CloudConfigFolder, "linode-build.packer")
+		if !utils.FileExists(buildContentFile) {
+			buildContentFile = path.Join(p.Opt.Env.CloudConfigFolder, "ln-build.packer")
+		}
+	default:
+		buildContentFile = path.Join(p.Opt.Env.CloudConfigFolder, "general-build.packer")
+	}
+
+	buildContent = utils.GetFileContent(buildContentFile)
+	if buildContent == "" {
+		errStr := fmt.Sprintf("Build file content not found at: %v", buildContentFile)
+		utils.ErrorF(errStr)
+		return fmt.Errorf(errStr)
+	}
+
+	buildContent = utils.RenderText(buildContent, p.ProviderConfig.BuildData)
+	buildFile := path.Join(p.ProviderConfig.ProviderFolder, "build.json")
+	p.ProviderConfig.BuildFile = buildFile
+	utils.WriteToFile(buildFile, buildContent)
+	utils.InforF("Write build provision of %s to: %s", color.HiYellowString(p.ProviderName), color.HiCyanString(buildFile))
+
+	// actually run building
+	err = p.Action(RunBuild)
+	if err != nil {
+		p.SnapshotFound = false
+		return err
+	}
+
+	err = p.Action(ListImage)
+	return err
+}
+
+// RunBuild run the packer command
+func (p *Provider) RunBuild() error {
+	packerBinary := fmt.Sprintf("%s/packer", p.Opt.Env.BinariesFolder)
+	if !utils.FileExists(packerBinary) {
+		packerBinary = "packer"
+	}
+
+	cmd := fmt.Sprintf("%s validate %s", packerBinary, p.ProviderConfig.BuildFile)
+	out, err := utils.RunCommandWithErr(cmd)
+	if err != nil {
+		utils.ErrorF(out)
+		return err
+	}
+	utils.InforF("The Packer file appears to be functioning properly: %s", color.HiCyanString(p.ProviderConfig.BuildFile))
+
+	// really start to build stuff here
+	utils.GoodF("Start packer build for: %s", color.HiCyanString(p.ProviderConfig.BuildFile))
+	cmd = fmt.Sprintf("%s build %s", packerBinary, p.ProviderConfig.BuildFile)
+	out, _ = utils.RunCommandWithErr(cmd)
+
+	if !strings.Contains(out, fmt.Sprintf("%v scan -f", libs.BINARY)) {
+		if !strings.Contains(out, fmt.Sprintf("%v: command not found", libs.BINARY)) {
+			utils.ErrorF(out)
+			return fmt.Errorf("error running provisioning")
+		}
+	}
+	return nil
+}

provider/parser.go

@@ -0,0 +1,127 @@
+package provider
+
+import (
+	"os"
+	"math/rand"
+	"time"
+
+	"github.com/Shopify/yaml"
+	"github.com/j3ssie/osmedeus/utils"
+)
+
+// ConfigProviders cloud config file
+type ConfigProviders struct {
+	Builder Builder          `yaml:"builder"`
+	Clouds  []ConfigProvider `yaml:"clouds"`
+}
+
+// Builder config for builder file
+type Builder struct {
+	BuildRepo string `yaml:"build_repo"`
+	PublicKey string `yaml:"public_key"`
+	SecretKey string `yaml:"secret_key"`
+}
+
+// ConfigProvider cloud config file for each provider from ~/osmedeus-base/cloud/provider.yaml
+type ConfigProvider struct {
+	// core part
+	Name  string `yaml:"name"`
+	Token string `yaml:"token"`
+
+	SecretKey   string `yaml:"secret_key"`
+	AccessKeyId string `yaml:"access_key"`
+
+	Provider     string `yaml:"provider"`
+	DefaultImage string `yaml:"default_image"`
+	Size         string `yaml:"size"`
+	Region       string `yaml:"region"`
+	Limit        int    `yaml:"limit"`
+	Username     string `yaml:"username"`
+
+	// BaseImage     string `yaml:"base"`
+	RedactedToken string `yaml:"-"`
+	Snapshot      string `yaml:"-"`
+	SnapshotID    string `yaml:"-"`
+	InstanceName  string `yaml:"-"`
+	PublicIP      string `yaml:"-"`
+	SshKey        string `yaml:"-"`
+
+	// for config
+	ProviderFolder string            `yaml:"-"`
+	ConfigFile     string            `yaml:"-"`
+	BuildFile      string            `yaml:"-"`
+	BuildData      map[string]string `yaml:"-"`
+
+	// for building
+	VarsFile     string `yaml:"-"`
+	RunnerFile   string `yaml:"-"`
+	BuildCommand string `yaml:"-"`
+	BaseFolder   string `yaml:"-"`
+	RawCommand   string `yaml:"-"`
+}
+
+// ParseProvider parse cloud file from ~/osmedeus-base/cloud/provider.yaml
+func ParseProvider(cloudFile string) (ConfigProviders, error) {
+	var clouds ConfigProviders
+	cloudFile = utils.NormalizePath(cloudFile)
+
+	yamlFile, err := os.ReadFile(cloudFile)
+	if err != nil {
+		utils.ErrorF("YAML parsing err #%v ", err)
+		return clouds, err
+	}
+	err = yaml.Unmarshal(yamlFile, &clouds)
+	if err != nil {
+		utils.ErrorF("Error: %v", err)
+		return clouds, err
+	}
+
+	return clouds, nil
+}
+
+// mics function
+
+const (
+	lowerChars  = "abcdefghijklmnopqrstuvwxyz"
+	upperChars  = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+	numberChars = "0123456789"
+	symbolChars = "!@#$%^&*()+-=[]{}<>?~"
+)
+
+func GeneratePassword(length int) string {
+	rand.Seed(time.Now().UnixNano())
+
+	var passwordChars []byte
+
+	// Add at least one of each character type
+	passwordChars = append(passwordChars, randomChar(lowerChars))
+	passwordChars = append(passwordChars, randomChar(upperChars))
+	passwordChars = append(passwordChars, randomChar(numberChars))
+	passwordChars = append(passwordChars, randomChar(symbolChars))
+
+	// Add remaining characters randomly
+	for i := len(passwordChars); i < length; i++ {
+		charType := rand.Intn(4) // 0 for lower, 1 for upper, 2 for number, 3 for symbol
+		switch charType {
+		case 0:
+			passwordChars = append(passwordChars, randomChar(lowerChars))
+		case 1:
+			passwordChars = append(passwordChars, randomChar(upperChars))
+		case 2:
+			passwordChars = append(passwordChars, randomChar(numberChars))
+		case 3:
+			passwordChars = append(passwordChars, randomChar(symbolChars))
+		}
+	}
+
+	// Shuffle the characters randomly
+	rand.Shuffle(len(passwordChars), func(i, j int) {
+		passwordChars[i], passwordChars[j] = passwordChars[j], passwordChars[i]
+	})
+
+	return string(passwordChars)
+}
+
+func randomChar(charset string) byte {
+	return charset[rand.Intn(len(charset))]
+}

provider/provider.go

@@ -0,0 +1,265 @@
+package provider
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/cenkalti/backoff/v4"
+	"github.com/fatih/color"
+	"github.com/j3ssie/osmedeus/libs"
+	"github.com/j3ssie/osmedeus/utils"
+	"github.com/spf13/cast"
+)
+
+type Provider struct {
+	ProviderName  string
+	Token         string
+	RedactedToken string
+	// for aws only
+	AccessKeyId       string
+	SecretKey         string
+	SecurityGroupID   string
+	SecurityGroupName string
+
+	Instances     []Instance `json:"-"`
+	InstanceLimit int
+	Available     bool
+	HealthCheck   bool
+
+	// for create snapshot
+	SnapshotID    string
+	SnapshotName  string
+	OldSnapShotID []string `json:"-"`
+	SnapshotFound bool
+	SSHKeyFound   bool
+	SSHPublicKey  string
+	SSHPrivateKey string
+	SSHKeyID      string
+	SSHUser       string
+
+	// for create
+	CreatedInstance Instance `json:"-"`
+	Region          string
+	Size            string
+	SSHKeyName      string
+
+	// mics
+	SwapSizeMap       map[string]int `json:"-"`
+	IsBackgroundCheck bool
+
+	// for building
+	ProviderConfig ConfigProvider `json:"-"`
+	Opt            libs.Options   `json:"-"`
+
+	// for retry
+	BackOff *backoff.ExponentialBackOff `json:"-"`
+	// client of vendor
+	Client interface{} `json:"-"`
+}
+
+type Instance struct {
+	InstanceID   string
+	InstanceName string
+	IPAddress    string
+
+	// meta data
+	Region    string
+	Size      string
+	Status    string
+	ImageID   string
+	ImageName string
+	CPU       string
+	// MB
+	Memory string
+	// GB
+	Disk string
+
+	// more for the osm
+	InputName    string
+	ProviderName string
+	CreatedAt    string
+}
+
+// InitProvider init provider object to easier interact with cloud provider
+func InitProvider(providerName string, token string) (Provider, error) {
+	var provider Provider
+	provider.ProviderName = providerName
+	provider.Token = token
+
+	if providerName == "aws" {
+		// token should be 'AccessKeyId,SecretKey'
+		provider.AccessKeyId = strings.TrimSpace(strings.Split(token, ",")[0])
+		provider.SecretKey = strings.TrimSpace(strings.Split(token, ",")[1])
+		provider.Token = token
+	}
+
+	provider.InitClient()
+	return provider, nil
+}
+
+// InitProviderWithConfig init provider object to easier interact with cloud provider
+func InitProviderWithConfig(opt libs.Options, providerConfig ConfigProvider) (Provider, error) {
+	var provider Provider
+	provider.ProviderName = providerConfig.Provider
+	provider.Token = providerConfig.Token
+
+	// for aws only
+	provider.AccessKeyId = providerConfig.AccessKeyId
+	provider.SecretKey = providerConfig.SecretKey
+	if provider.AccessKeyId != "" {
+		provider.Token = provider.AccessKeyId + "," + provider.SecretKey
+	}
+
+	provider.ProviderConfig = providerConfig
+	provider.Opt = opt
+	if opt.Cloud.BackgroundRun {
+		provider.IsBackgroundCheck = true
+	}
+
+	if err := provider.InitClient(); err != nil {
+		return provider, fmt.Errorf("unable to validate token: %v", provider.Token)
+	}
+
+	provider.Prepare()
+	return provider, nil
+}
+
+func (p *Provider) InitClient() (err error) {
+	if p.Token == "" && p.AccessKeyId == "" {
+		utils.ErrorF("empty or invalid token: %v", p.Token)
+		return fmt.Errorf("empty or invalid token")
+	}
+	if len(p.Token) > 5 {
+		p.RedactedToken = p.Token[:5] + "***" + p.Token[len(p.Token)-5:len(p.Token)]
+		if !p.IsBackgroundCheck {
+			utils.InforF("Init %v provider with token: %v", color.HiYellowString(p.ProviderName), color.HiCyanString(p.RedactedToken))
+		}
+	}
+
+	p.Available = true
+	switch p.ProviderName {
+	case "do", "digitalocean":
+		p.ClientDO()
+	case "ln", "line", "linode":
+		p.ClientLinode()
+	case "aw", "aws", "asw":
+		p.ClientAWS()
+	default:
+		p.ClientDO()
+	}
+
+	// skip balance check if health check
+	if p.HealthCheck {
+		return nil
+	}
+
+	switch p.ProviderName {
+	case "do", "digitalocean":
+		err = p.AccountDO()
+	case "ln", "line", "linode":
+		err = p.AccountLN()
+	case "aw", "aws", "asw":
+		err = p.AccountAWS()
+	default:
+		err = p.AccountDO()
+	}
+
+	return err
+}
+
+// Prepare setup some default variables
+func (p *Provider) Prepare() {
+	// get snapshot
+	version := strings.ReplaceAll(strings.TrimSpace(libs.VERSION), " ", "-")
+	SnapshotName := fmt.Sprintf("%s-base-%s", strings.TrimSpace(libs.SNAPSHOT), version)
+	p.SnapshotName = SnapshotName
+
+	// sshKey
+	keyName := fmt.Sprintf("%s-cloud-key", strings.TrimSpace(libs.SNAPSHOT))
+	p.SSHKeyName = keyName
+
+	// for retry
+	b := backoff.NewExponentialBackOff()
+	// It never stops if MaxElapsedTime == 0.
+	b.MaxElapsedTime = 1200 * time.Second
+	b.Multiplier = 2.0
+	b.InitialInterval = 30 * time.Second
+	p.BackOff = b
+
+	if p.Opt.Cloud.Retry > 0 {
+		b.MaxElapsedTime = time.Duration(p.Opt.Cloud.Retry*60) * time.Second
+	}
+
+	// setup ssh key
+	if p.SSHPublicKey == "" {
+		p.SSHPublicKey = p.Opt.Cloud.PublicKeyContent
+	}
+	if p.SSHPrivateKey == "" {
+		p.SSHPrivateKey = p.Opt.Cloud.SecretKeyContent
+	}
+
+	utils.DebugF("Get data of cloud provider")
+	switch p.ProviderName {
+	case "do", "digitalocean":
+		p.DefaultDO()
+	case "ln", "line", "linode":
+		p.DefaultLinode()
+	case "aw", "aws", "asw":
+		p.DefaultAWS()
+	default:
+		p.DefaultDO()
+	}
+
+	p.Action(GetSSHKey)
+	p.Action(ListImage)
+
+	if p.SSHKeyID != "" {
+		if !p.IsBackgroundCheck {
+			utils.InforF("Found SSH Key ID: %v", color.HiBlueString(p.SSHKeyID))
+		}
+	}
+}
+
+func (p *Provider) Action(actionName string, params ...interface{}) error {
+	var err error
+	var param interface{}
+	if len(params) > 0 {
+		param = params[0]
+	}
+
+	if !p.IsBackgroundCheck {
+		utils.InforF("[%v] running action: %v", p.ProviderName, color.HiBlueString(actionName))
+	}
+
+	operation := func() error {
+		switch actionName {
+		case GetSSHKey:
+			err = p.GetSSHKey()
+		case ListInstance:
+			err = p.ListInstance()
+		case ListImage:
+			err = p.ListSnapShot()
+		case RunBuild:
+			err = p.RunBuild()
+		case GetInstanceInfo:
+			err = p.GetInstanceInfo(param)
+		case BootInstance:
+			err = p.BootInstance(param)
+		case CreateInstance:
+			err = p.CreateInstance(cast.ToString(param))
+		default:
+			err = p.ListInstance()
+		}
+		if err != nil {
+			utils.ErrorF("error running action %v: %v", color.HiCyanString(actionName), err)
+		}
+		return err
+	}
+	err = backoff.Retry(operation, p.BackOff)
+	if err != nil {
+		utils.ErrorF("error running action %v -- %v", actionName, p.ProviderName)
+		return err
+	}
+	return nil
+}

provider/provider_aws.go

@@ -0,0 +1,516 @@
+package provider
+
+import (
+	"crypto/sha256"
+	"encoding/base64"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/spf13/cast"
+	"golang.org/x/crypto/ssh"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/costexplorer"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/fatih/color"
+	"github.com/j3ssie/osmedeus/libs"
+	"github.com/j3ssie/osmedeus/utils"
+)
+
+// DefaultAWS set some default data for AWS provider
+func (p *Provider) DefaultAWS() {
+	p.Region = "ap-southeast-1"
+	p.Size = "t2.medium"
+	p.SecurityGroupName = "osmp-allow-root-access"
+	p.SSHUser = p.ProviderConfig.Username
+
+	if p.ProviderConfig.Username != "" {
+		p.SSHUser = "admin"
+	}
+	if p.ProviderConfig.Region != "" {
+		p.Region = p.ProviderConfig.Region
+	}
+	if p.ProviderConfig.Size != "" {
+		p.Size = p.ProviderConfig.Size
+	}
+}
+
+func (p *Provider) InitSessionAWS() (*session.Session, error) {
+	sess, err := session.NewSession(&aws.Config{
+		Region:      aws.String(p.Region),
+		Credentials: credentials.NewStaticCredentials(p.AccessKeyId, p.SecretKey, ""),
+	})
+
+	return sess, err
+}
+
+func (p *Provider) ClientAWS() {
+	client, err := p.InitSessionAWS()
+	if err != nil {
+		panic(err)
+	}
+	p.ProviderName = "aws"
+	p.Client = client
+}
+
+func (p *Provider) ConvertClientAWS() *session.Session {
+	sess, ok := p.Client.(*session.Session)
+	if !ok {
+		utils.ErrorF("error converting aws session %v", ok)
+	}
+	sess.Config.Region = aws.String(p.Region)
+	return sess
+}
+
+func (p *Provider) AccountAWS() error {
+	ceSvc := costexplorer.New(p.ConvertClientAWS())
+	// Set the parameters for the query
+	now := time.Now()
+	start := time.Date(now.Year(), now.Month()-1, 1, 0, 0, 0, 0, now.Location())
+	end := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, now.Location())
+	params := &costexplorer.GetCostAndUsageInput{
+		TimePeriod: &costexplorer.DateInterval{
+			Start: aws.String(start.Format("2006-01-02")),
+			End:   aws.String(end.Format("2006-01-02")),
+		},
+		Granularity: aws.String(costexplorer.GranularityMonthly),
+		Metrics:     []*string{aws.String("UnblendedCost")},
+	}
+
+	// Send the query and get the results
+	// result, err := ceSvc.GetCostAndUsage(params)
+	// if err != nil {
+	// 	utils.ErrorF("Error getting cost and usage:", err)
+	// 	return err
+	// }
+
+	// Send the query and get the results
+	result, err := ceSvc.GetCostAndUsage(params)
+	if err != nil {
+		utils.ErrorF("Error getting cost and usage: %v", err)
+		return err
+	}
+
+	// Print the total cost for the previous month
+	cost := *result.ResultsByTime[0].Total["UnblendedCost"].Amount
+
+	if !p.IsBackgroundCheck {
+		utils.InforF("The total cost of AWS services for the this month was %s", color.HiRedString("$"+cost))
+	}
+	return nil
+}
+
+func (p *Provider) GetSSHKeyAWS() error {
+	ec2Svc := ec2.New(p.ConvertClientAWS())
+	// Get a list of all the key pairs in the account
+	keyPairsOutput, err := ec2Svc.DescribeKeyPairs(&ec2.DescribeKeyPairsInput{})
+	if err != nil {
+		utils.ErrorF("Error describing key pairs: %v", err)
+		return err
+	}
+
+	pubKeyBytes := []byte(p.SSHPublicKey)
+	// Parse the key, other info ignored
+	pubKey, _, _, _, err := ssh.ParseAuthorizedKey(pubKeyBytes)
+	if err != nil {
+		utils.ErrorF("%v", err)
+		return err
+	}
+	hash := sha256.Sum256(pubKey.Marshal())
+	sshHash := base64.StdEncoding.EncodeToString(hash[:])
+
+	// Check if your SSH key is present in the list
+	for _, keyPair := range keyPairsOutput.KeyPairs {
+		// found the same key name but different key fingerprint
+		if *keyPair.KeyName == p.SSHKeyName && *keyPair.KeyFingerprint != sshHash {
+			// Delete the key pair
+			_, err := ec2Svc.DeleteKeyPair(&ec2.DeleteKeyPairInput{
+				KeyName: aws.String(*keyPair.KeyName),
+			})
+			if err != nil {
+				utils.ErrorF("%v", err)
+			}
+			utils.InforF("Successfully deleted key pair %s", color.HiBlueString(*keyPair.KeyName))
+		}
+
+		if *keyPair.KeyFingerprint == sshHash {
+			p.SSHKeyID = cast.ToString(*keyPair.KeyPairId)
+			p.SSHKeyFound = true
+			break
+		}
+
+	}
+
+	if p.SSHKeyFound {
+		utils.DebugF("Your SSH key was found in the account: %v -- %v", color.HiCyanString(p.SSHKeyName), color.HiCyanString(p.SSHKeyID))
+		return nil
+	}
+
+	// Import the SSH key into your AWS account
+	result, err := ec2Svc.ImportKeyPair(&ec2.ImportKeyPairInput{
+		KeyName:           aws.String(p.SSHKeyName),
+		PublicKeyMaterial: []byte(p.SSHPublicKey),
+	})
+	if err != nil {
+		utils.ErrorF("Error create key pairs: %v", err)
+		return err
+	}
+	utils.DebugF("Successfully imported SSH key: %v -- %v", color.HiCyanString(*result.KeyName), color.HiCyanString(*result.KeyPairId))
+	p.SSHKeyID = cast.ToString(*result.KeyPairId)
+	p.SSHKeyFound = true
+
+	return nil
+}
+
+func (p *Provider) ListSnapshotAWS() error {
+	svc := ec2.New(p.ConvertClientAWS())
+
+	// listing only image that own by you
+	self := "self"
+	ownImages := &ec2.DescribeImagesInput{Owners: []*string{&self}}
+	result, err := svc.DescribeImages(ownImages)
+	if err != nil {
+		utils.ErrorF("err: Unable to list images, %v", err)
+		return err
+	}
+
+	for _, item := range result.Images {
+		if strings.HasPrefix(*item.Name, libs.SNAPSHOT) {
+			p.OldSnapShotID = append(p.OldSnapShotID, *item.ImageId)
+		}
+
+		if strings.TrimSpace(*item.Name) == strings.TrimSpace(p.SnapshotName) {
+			utils.DebugF("Found base image snapshot with ID: %s", color.HiBlueString(*item.ImageId))
+			p.SnapshotID = *item.ImageId
+			p.SnapshotName = *item.Name
+			p.SnapshotFound = true
+		}
+	}
+
+	return nil
+}
+
+func (p *Provider) DeleteImageAWS(id string) error {
+	if p.SnapshotID == "" {
+		return nil
+	}
+	svc := ec2.New(p.ConvertClientAWS())
+	deletedImage := &ec2.DeregisterImageInput{ImageId: &p.SnapshotID}
+	_, err := svc.DeregisterImage(deletedImage)
+	if err != nil {
+		utils.ErrorF("err: Unable to delete snapshot: %v -- %v", id, err)
+		return err
+	}
+	utils.InforF("Deleted image ID: %v", color.HiRedString(p.SnapshotID))
+	p.DeleteSnapshotAWS()
+	return nil
+}
+
+func (p *Provider) DeleteSnapshotAWS() error {
+	svc := ec2.New(p.ConvertClientAWS())
+
+	// List all snapshots
+	result, err := svc.DescribeSnapshots(&ec2.DescribeSnapshotsInput{
+		OwnerIds:   []*string{aws.String("self")},
+		MaxResults: aws.Int64(1000),
+	})
+	if err != nil {
+		utils.ErrorF("Error listing snapshots: %v", err)
+		return err
+	}
+
+	var snapshotIDs []string
+	for _, snapshot := range result.Snapshots {
+
+		for _, tag := range snapshot.Tags {
+			if *tag.Key == "Name" && *tag.Value == "Osmedeus Premium Image" {
+				snapshotIDs = append(snapshotIDs, *snapshot.SnapshotId)
+			}
+		}
+	}
+
+	for _, snapshotID := range snapshotIDs {
+		_, err = svc.DeleteSnapshot(&ec2.DeleteSnapshotInput{
+			SnapshotId: aws.String(snapshotID),
+		})
+		if err != nil {
+			utils.ErrorF("Error deleting snapshot: %v -- %v", snapshotID, err)
+		} else {
+			utils.DebugF("Delted snapshot ID: %v", color.HiRedString(snapshotID))
+		}
+	}
+
+	return nil
+}
+
+func (p *Provider) ListInstanceAWS() error {
+	svc := ec2.New(p.ConvertClientAWS())
+	result, err := svc.DescribeInstances(nil)
+	if err != nil {
+		utils.ErrorF("err: Unable to list ec2 instances: %v", err)
+		return err
+	}
+
+	var numberOfInstance int
+	for i := range result.Reservations {
+		for _, instance := range result.Reservations[i].Instances {
+			if *instance.State.Name != "running" {
+				continue
+			}
+			numberOfInstance += 1
+
+			launchTime := *instance.LaunchTime
+			creationDate := launchTime.Format(time.RFC1123)
+			parsedInstance := Instance{
+				InstanceID:   cast.ToString(*instance.InstanceId),
+				IPAddress:    *instance.PublicIpAddress,
+				InstanceName: *instance.State.Name,
+				ImageID:      cast.ToString(*instance.ImageId),
+				// ImageName: instance.Image.Name,
+				// Region: instance.Region.Slug,
+				// Region:    *instance.Architecture,
+				// Memory:       cast.ToString(instance.Memory),
+				// CPU:          cast.ToString(instance.Vcpus),
+				// Disk:         cast.ToString(instance.Disk),
+				Status:       *instance.State.Name,
+				CreatedAt:    cast.ToString(creationDate),
+				InputName:    "",
+				ProviderName: "aws",
+			}
+
+			p.Instances = append(p.Instances, parsedInstance)
+		}
+	}
+
+	utils.InforF("Found %v running instances", color.HiMagentaString("%v", numberOfInstance))
+	// check if we reach max instance number
+	if p.InstanceLimit > 0 {
+		if len(p.Instances) >= p.InstanceLimit {
+			p.Available = false
+		}
+	}
+
+	return nil
+}
+
+func (p *Provider) CreateInstanceAWS(InstanceName string) (instanctID string, err error) {
+	svc := ec2.New(p.ConvertClientAWS())
+	p.CreateSecurityGroup()
+
+	// Set the parameters for the instance
+	params := &ec2.RunInstancesInput{
+		// InstanceName: aws.String(name),
+		ImageId:      aws.String(p.SnapshotID), // Replace with the ID of the image you want to use
+		InstanceType: aws.String(p.Size),       // Specify the instance type like t2.micro
+		MinCount:     aws.Int64(1),
+		MaxCount:     aws.Int64(1),
+		KeyName:      aws.String(p.SSHKeyName),
+		SecurityGroups: []*string{
+			aws.String(p.SecurityGroupName),
+		},
+		TagSpecifications: []*ec2.TagSpecification{
+			{
+				ResourceType: aws.String("instance"),
+				Tags: []*ec2.Tag{
+					{
+						Key:   aws.String("Name"),
+						Value: aws.String(InstanceName),
+					},
+				},
+			},
+		},
+	}
+
+	// Create the instance
+	result, err := svc.RunInstances(params)
+	if err != nil {
+		utils.ErrorF("Error creating instance: %v", err)
+		return
+	}
+
+	// Get the instance ID
+	instanctID = *result.Instances[0].InstanceId
+	utils.InforF("Successfully Created Instance ID: %v -- %v", color.HiBlueString(instanctID), color.HiBlueString(InstanceName))
+	utils.DebugF("Waiting for the instance %v to be ready...", color.HiBlueString(instanctID))
+
+	time.Sleep(60 * time.Second)
+	// Get the instance state
+	for i := 0; i < 10; i++ {
+		if p.InstanceReady(instanctID) == nil {
+			return instanctID, nil
+		}
+		time.Sleep(60 * time.Second)
+	}
+
+	return instanctID, nil
+}
+
+func (p *Provider) InstanceReady(instanceID string) error {
+	svc := ec2.New(p.ConvertClientAWS())
+
+	// Describe the instance
+	params := &ec2.DescribeInstancesInput{
+		InstanceIds: []*string{
+			aws.String(instanceID),
+		},
+	}
+	_, err := svc.DescribeInstances(params)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (p *Provider) InstanceReboot(instanceID string) error {
+	svc := ec2.New(p.ConvertClientAWS())
+
+	// Create the input for the RebootInstances operation
+	params := &ec2.RebootInstancesInput{
+		InstanceIds: []*string{aws.String(instanceID)},
+	}
+
+	// Call the RebootInstances operation
+	_, err := svc.RebootInstances(params)
+	if err != nil {
+		fmt.Println("Error rebooting instance:", err)
+		return err
+	}
+	return nil
+}
+
+func (p *Provider) DeleteInstanceAWS(id string) error {
+	svc := ec2.New(p.ConvertClientAWS())
+
+	// Set the parameters for the instance
+	params := &ec2.TerminateInstancesInput{
+		InstanceIds: []*string{
+			aws.String(id), // Replace with the ID of the instance you want to delete
+		},
+	}
+
+	// Delete the instance
+	_, err := svc.TerminateInstances(params)
+	if err != nil {
+		utils.ErrorF("Error deleting instance: %v", err)
+		return err
+	}
+
+	utils.InforF("Successfully Deleted instance ID: %v", color.HiRedString(id))
+	return nil
+}
+
+func (p *Provider) InstanceInfoAWS(id string) (Instance, error) {
+	var parsedInstance Instance
+
+	svc := ec2.New(p.ConvertClientAWS())
+
+	// Set the parameters for the instance
+	params := &ec2.DescribeInstancesInput{
+		InstanceIds: []*string{
+			aws.String(id),
+		},
+	}
+
+	// Get the instance information
+	result, err := svc.DescribeInstances(params)
+	if err != nil {
+		utils.ErrorF("Error getting instance information: %v", err)
+		return parsedInstance, err
+	}
+
+	// Print the instance information
+	for _, reservation := range result.Reservations {
+		for _, instance := range reservation.Instances {
+
+			launchTime := *instance.LaunchTime
+			creationDate := launchTime.Format(time.RFC1123)
+
+			var instanceName string
+			for _, tag := range instance.Tags {
+				if *tag.Key == "Name" {
+					instanceName = *tag.Value
+					break
+				}
+			}
+
+			parsedInstance = Instance{
+				InstanceID:   cast.ToString(*instance.InstanceId),
+				IPAddress:    *instance.PublicIpAddress,
+				InstanceName: instanceName,
+				ImageID:      cast.ToString(*instance.ImageId),
+				Status:       *instance.State.Name,
+				CreatedAt:    cast.ToString(creationDate),
+				InputName:    "",
+				ProviderName: "aws",
+			}
+
+		}
+	}
+
+	p.CreatedInstance = parsedInstance
+	utils.DebugF("Instance ID Info: %v -- %v -- %v", color.HiBlueString(p.CreatedInstance.InstanceID), p.CreatedInstance.InstanceName, p.CreatedInstance.IPAddress)
+	return parsedInstance, nil
+}
+
+func (p *Provider) CreateSecurityGroup() error {
+	svc := ec2.New(p.ConvertClientAWS())
+
+	// Set the parameters for the security group
+	params := &ec2.DescribeSecurityGroupsInput{
+		GroupNames: []*string{
+			aws.String(p.SecurityGroupName), // Replace with the ID of the security group you want to check
+		},
+	}
+
+	// Get the security group information
+	scGroups, err := svc.DescribeSecurityGroups(params)
+	if err == nil {
+		// Print the security group information
+		for _, group := range scGroups.SecurityGroups {
+			if *group.GroupName == p.SecurityGroupName {
+				p.SecurityGroupID = *group.GroupId
+				utils.DebugF("Security Group allow root access has been found: %v", color.HiBlueString(p.SecurityGroupID))
+				return nil
+			}
+		}
+	}
+
+	// only create if not found
+
+	// Create the security group
+	result, err := svc.CreateSecurityGroup(&ec2.CreateSecurityGroupInput{
+		GroupName:   aws.String("osmp-allow-root-access"),
+		Description: aws.String("Security group for allowing root access to EC2 instances"),
+	})
+	if err != nil {
+		utils.ErrorF("Error creating security group: %v", err)
+		return err
+	}
+
+	// Add a rule to the security group to allow SSH access from any IP address
+	_, err = svc.AuthorizeSecurityGroupIngress(&ec2.AuthorizeSecurityGroupIngressInput{
+		GroupId: aws.String(*result.GroupId),
+		IpPermissions: []*ec2.IpPermission{
+			{
+				FromPort:   aws.Int64(22),
+				ToPort:     aws.Int64(22),
+				IpProtocol: aws.String("tcp"),
+				IpRanges: []*ec2.IpRange{
+					{CidrIp: aws.String("0.0.0.0/0")},
+				},
+			},
+		},
+	})
+	if err != nil {
+		utils.ErrorF("Error adding rule to security group: %v", err)
+		return err
+	}
+
+	p.SecurityGroupID = *result.GroupId
+	utils.DebugF("Security Group allow root access has been found: %v", color.HiBlueString(p.SecurityGroupID))
+	return nil
+}

provider/provider_digitalocean.go

@@ -0,0 +1,294 @@
+package provider
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"strings"
+	"time"
+
+	"github.com/digitalocean/godo"
+	"github.com/fatih/color"
+	"github.com/j3ssie/osmedeus/libs"
+	"github.com/j3ssie/osmedeus/utils"
+	"github.com/spf13/cast"
+)
+
+// DefaultDO set some default data for DO provider
+func (p *Provider) DefaultDO() {
+	p.Region = "sfo3"
+	p.Size = "s-2vcpu-4gb"
+	p.SSHUser = p.ProviderConfig.Username
+
+	if p.ProviderConfig.Username != "" {
+		p.SSHUser = "root"
+	}
+	if p.ProviderConfig.Region != "" {
+		p.Region = p.ProviderConfig.Region
+	}
+	if p.ProviderConfig.Size != "" {
+		p.Size = p.ProviderConfig.Size
+	}
+}
+
+func (p *Provider) ClientDO() {
+	client := godo.NewFromToken(p.Token)
+	p.Client = client
+}
+
+func (p *Provider) ConvertClientDO() *godo.Client {
+	client, ok := p.Client.(*godo.Client)
+	if !ok {
+		utils.ErrorF("error converting digital ocean session %v", ok)
+	}
+	return client
+}
+
+func (p *Provider) AccountDO() error {
+	client := p.ConvertClientDO()
+	ctx := context.TODO()
+
+	account, _, err := client.Account.Get(ctx)
+	if err != nil {
+		return fmt.Errorf("error getting account information")
+	}
+	p.InstanceLimit = account.DropletLimit
+
+	bill, _, err := client.Balance.Get(ctx)
+	if err != nil {
+		return fmt.Errorf("error getting account information")
+	}
+
+	if !p.IsBackgroundCheck {
+		utils.InforF("Account Billing Information: MonthToDateBalance: %v -- AccountBalance: %v", strings.TrimLeft(color.HiRedString(bill.MonthToDateBalance), "-"), color.HiGreenString(bill.AccountBalance))
+	}
+
+	return nil
+}
+
+func (p *Provider) ListInstanceDO() error {
+	client := p.ConvertClientDO()
+
+	ctx := context.TODO()
+	opt := &godo.ListOptions{
+		Page:    1,
+		PerPage: 1000,
+	}
+
+	droplets, _, err := client.Droplets.List(ctx, opt)
+	if err != nil {
+		return fmt.Errorf("error getting digital ocean instance")
+	}
+	utils.DebugF("found %v instances", len(droplets))
+
+	for _, instance := range droplets {
+		ipAddress, ok := instance.PublicIPv4()
+		if ok != nil || ipAddress == "" {
+			utils.ErrorF("Instance has no public IP: %v -- %v", instance.ID, instance.Name)
+			continue
+		}
+		parsedInstance := Instance{
+			InstanceID:   cast.ToString(instance.ID),
+			IPAddress:    ipAddress,
+			InstanceName: instance.Name,
+			ImageID:      cast.ToString(instance.Image.ID),
+			ImageName:    instance.Image.Name,
+			Region:       instance.Region.Slug,
+			Memory:       cast.ToString(instance.Memory),
+			CPU:          cast.ToString(instance.Vcpus),
+			Disk:         cast.ToString(instance.Disk),
+			Status:       instance.Status,
+			CreatedAt:    instance.Created,
+
+			InputName:    "",
+			ProviderName: "do",
+		}
+
+		p.Instances = append(p.Instances, parsedInstance)
+	}
+
+	// check if we reach max instance number
+	if p.InstanceLimit > 0 {
+		if len(p.Instances) >= p.InstanceLimit {
+			p.Available = false
+		}
+	}
+
+	return nil
+}
+
+func (p *Provider) GetSSHKeyDO() error {
+	client := p.ConvertClientDO()
+
+	ctx := context.TODO()
+	opt := &godo.ListOptions{
+		Page:    1,
+		PerPage: 1000,
+	}
+
+	keys, _, err := client.Keys.List(ctx, opt)
+	if err != nil {
+		return fmt.Errorf("error listing ssh key -- %v", err)
+	}
+
+	for _, key := range keys {
+		if strings.TrimSpace(key.PublicKey) == strings.TrimSpace(p.SSHPublicKey) {
+			p.SSHKeyID = cast.ToString(key.ID)
+			p.SSHKeyFound = true
+			utils.DebugF("Found SSH Key: %v -- %v ", color.HiCyanString(key.Name), color.HiCyanString(p.SSHKeyID))
+		}
+	}
+
+	// create one if not found
+	if !p.SSHKeyFound {
+		utils.DebugF("No SSHKey found. create a new one")
+		createRequest := &godo.KeyCreateRequest{
+			Name:      p.SSHKeyName,
+			PublicKey: p.SSHPublicKey,
+		}
+		transfer, _, err := client.Keys.Create(ctx, createRequest)
+		if err == nil {
+			p.SSHKeyID = cast.ToString(transfer.ID)
+			p.SSHKeyFound = true
+			utils.DebugF("Created new SSH Key: %v", color.HiCyanString(p.SSHKeyID))
+		} else {
+			return fmt.Errorf("error create ssh key -- %v", err)
+		}
+	}
+
+	return nil
+}
+
+func (p *Provider) ListSnapshotDO() error {
+	client := p.ConvertClientDO()
+
+	ctx := context.TODO()
+	opt := &godo.ListOptions{
+		Page:    1,
+		PerPage: 1000,
+	}
+
+	snapshots, _, err := client.Snapshots.List(ctx, opt)
+	if err != nil {
+		return fmt.Errorf("error getting digital ocean snapshot")
+	}
+	for _, instance := range snapshots {
+		name := instance.Name
+		id := cast.ToString(instance.ID)
+
+		if strings.HasPrefix(name, libs.SNAPSHOT) {
+			p.OldSnapShotID = append(p.OldSnapShotID, id)
+		}
+
+		if strings.TrimSpace(name) == strings.TrimSpace(p.SnapshotName) {
+			utils.DebugF("Found base image snapshot with ID: %s", color.HiBlueString(id))
+			p.SnapshotID = id
+			p.SnapshotName = name
+			p.SnapshotFound = true
+
+		}
+	}
+
+	return nil
+}
+
+func (p *Provider) CreateInstanceDO(name string) (dropletId int, err error) {
+	client := p.ConvertClientDO()
+
+	ctx := context.TODO()
+	createRequest := &godo.DropletCreateRequest{
+		Name:   name,
+		Region: p.Region,
+		Size:   p.Size,
+		Image: godo.DropletCreateImage{
+			ID: cast.ToInt(p.SnapshotID),
+			//Slug: "ubuntu-16-04-x64",
+		},
+		// SSHKeys: []godo.DropletCreateSSHKey{
+		// 	godo.DropletCreateSSHKey{ID: cast.ToInt(p.SSHKeyID)},
+		// },
+		// Tags: []string{libs.SNAPSHOT},
+
+		SSHKeys: []godo.DropletCreateSSHKey{
+			{ID: cast.ToInt(p.SSHKeyID)},
+		},
+		Tags: []string{libs.SNAPSHOT},
+	}
+
+	instance, res, err := client.Droplets.Create(ctx, createRequest)
+	if err != nil {
+		utils.ErrorF("error create digital ocean instance -- %v", err)
+		content, ok := io.ReadAll(res.Body)
+		if ok == nil {
+			fmt.Println(string(content))
+		}
+		return dropletId, fmt.Errorf("error create digital ocean instance -- %v", err)
+	}
+
+	// get droplet IP info
+	dropletId = instance.ID
+	utils.DebugF("Created instance %v", color.HiBlueString("%v", instance.ID))
+	utils.DebugF("Waiting for the instance %v to be ready...", color.HiBlueString("%v", instance.ID))
+	time.Sleep(100 * time.Second)
+	return dropletId, nil
+}
+
+func (p *Provider) InstanceInfoDO(id int) (Instance, error) {
+	var parsedInstance Instance
+	client := p.ConvertClientDO()
+	ctx := context.TODO()
+	instance, _, err := client.Droplets.Get(ctx, id)
+	if err != nil {
+		return parsedInstance, fmt.Errorf("error get instance info:")
+	}
+
+	ipAddress, err := instance.PublicIPv4()
+	if err != nil || ipAddress == "" {
+		return parsedInstance, fmt.Errorf("no public ip address yet")
+	}
+	parsedInstance = Instance{
+		InstanceID:   cast.ToString(instance.ID),
+		IPAddress:    ipAddress,
+		InstanceName: instance.Name,
+		ImageID:      cast.ToString(instance.Image.ID),
+		ImageName:    instance.Image.Name,
+		Region:       instance.Region.Slug,
+		Memory:       cast.ToString(instance.Memory),
+		CPU:          cast.ToString(instance.Vcpus),
+		Disk:         cast.ToString(instance.Disk),
+		Status:       instance.Status,
+		CreatedAt:    instance.Created,
+
+		InputName:    "",
+		ProviderName: "do",
+	}
+	p.CreatedInstance = parsedInstance
+	utils.DebugF("Successfully Created Instance: %v -- %v -- %v", p.CreatedInstance.InstanceID, p.CreatedInstance.InstanceName, p.CreatedInstance.IPAddress)
+
+	return parsedInstance, nil
+}
+
+func (p *Provider) DeleteInstanceDO(id string) error {
+	client := p.ConvertClientDO()
+	ctx := context.TODO()
+	_, err := client.Droplets.Delete(ctx, cast.ToInt(id))
+	if err != nil {
+		utils.ErrorF("error delete instance -- %v", err)
+		return fmt.Errorf("error delete instance")
+	}
+	utils.InforF("Successfully Deleted instance ID: %v", color.HiRedString(id))
+	return nil
+}
+
+func (p *Provider) DeleteSnapShotDO(id string) error {
+	client := p.ConvertClientDO()
+	ctx := context.TODO()
+
+	_, err := client.Snapshots.Delete(ctx, id)
+	if err != nil {
+		utils.ErrorF("error delete snapshot -- %v", err)
+		return fmt.Errorf("error delete instance")
+	}
+	utils.InforF("Deleted snapshot ID: %v", color.HiRedString(id))
+	return nil
+}

provider/provider_linode.go

@@ -0,0 +1,477 @@
+package provider
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/fatih/color"
+	"github.com/j3ssie/osmedeus/libs"
+	"github.com/j3ssie/osmedeus/utils"
+	"github.com/linode/linodego"
+	"github.com/spf13/cast"
+	"golang.org/x/oauth2"
+)
+
+// DefaultLinode set some default data for DO provider
+func (p *Provider) DefaultLinode() {
+	p.Region = "us-east"
+	p.Size = "g6-standard-1"
+	p.SSHUser = p.ProviderConfig.Username
+
+	if p.ProviderConfig.Username != "" {
+		p.SSHUser = "root"
+	}
+	if p.ProviderConfig.Region != "" {
+		p.Region = p.ProviderConfig.Region
+	}
+	if p.ProviderConfig.Size != "" {
+		p.Size = p.ProviderConfig.Size
+	}
+
+	if p.Opt.Cloud.Size != "" {
+		p.Size = p.Opt.Cloud.Size
+	}
+	if p.Opt.Cloud.Region != "" {
+		p.Region = p.Opt.Cloud.Region
+	}
+
+	p.LinodeDiskMap()
+}
+
+func (p *Provider) ClientLinode() error {
+	tokenSource := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: p.Token})
+	oauth2Client := &http.Client{
+		Transport: &oauth2.Transport{
+			Source: tokenSource,
+		},
+	}
+
+	linodeClient := linodego.NewClient(oauth2Client)
+
+	p.Client = linodeClient
+	return nil
+}
+
+func (p *Provider) ConvertClientLinode() linodego.Client {
+	client, ok := p.Client.(linodego.Client)
+	if !ok {
+		utils.ErrorF("error converting linode session %v", ok)
+	}
+	return client
+}
+
+func (p *Provider) AccountLN() error {
+	client := p.ConvertClientLinode()
+	ctx := context.TODO()
+	account, err := client.GetAccount(ctx)
+	if err != nil {
+		return fmt.Errorf("error getting account information")
+	}
+	if !p.IsBackgroundCheck {
+		utils.InforF("Account Billing Information: BalanceUninvoiced: %v -- AccountBalance: %v", color.HiRedString("%v", account.BalanceUninvoiced), color.HiGreenString("%v", account.Balance))
+	}
+	helper := false
+	opt := linodego.AccountSettingsUpdateOptions{
+		NetworkHelper: &helper,
+	}
+
+	// @TODO: no idea why this function false
+	// client.UpdateAccountSettings(context.Background(), opt)
+
+	req := client.R(ctx).SetResult(&linodego.AccountSettings{})
+	if bodyData, err := json.Marshal(&opt); err == nil {
+		req.URL = "https://api.linode.com/v4/account/settings"
+		//fmt.Println("err marshal", err)
+		body := string(bodyData)
+		req.SetBody(body).Put(req.URL)
+	}
+	//}
+	return err
+}
+
+// LinodeTest list all instances
+func (p *Provider) LinodeTest() error {
+	linodeClient := p.ConvertClientLinode()
+
+	res, err := linodeClient.GetInstance(context.Background(), 4090913)
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Printf("%v", res)
+
+	return nil
+}
+
+func (p *Provider) ListInstanceLN() error {
+	client := p.ConvertClientLinode()
+
+	opt := &linodego.ListOptions{
+		PageOptions: nil,
+		Filter:      "",
+	}
+
+	instances, err := client.ListInstances(context.TODO(), opt)
+	if err != nil {
+		return fmt.Errorf("error getting linode instance")
+	}
+
+	for _, instance := range instances {
+		//instance.IPv4
+		if len(instance.IPv4) == 0 {
+			utils.ErrorF("Instance has no public IP: %v -- %v", instance.ID, instance.Label)
+			continue
+		}
+		ipAddress := instance.IPv4[0]
+
+		parsedInstance := Instance{
+			InstanceID:   cast.ToString(instance.ID),
+			IPAddress:    cast.ToString(ipAddress),
+			InstanceName: instance.Label,
+			ImageID:      cast.ToString(instance.Image),
+			ImageName:    instance.Image,
+			Region:       instance.Region,
+			Memory:       cast.ToString(instance.Specs.Memory),
+			CPU:          cast.ToString(instance.Specs.VCPUs),
+			Disk:         cast.ToString(instance.Specs.Disk),
+			Status:       cast.ToString(instance.Status),
+			CreatedAt:    cast.ToString(instance.Created),
+
+			InputName:    "",
+			ProviderName: "do",
+		}
+
+		p.Instances = append(p.Instances, parsedInstance)
+	}
+	return nil
+}
+
+func (p *Provider) GetSSHKeyLN() error {
+	client := p.ConvertClientLinode()
+	ctx := context.TODO()
+	opt := &linodego.ListOptions{
+		PageOptions: nil,
+		Filter:      "",
+	}
+
+	keys, err := client.ListSSHKeys(ctx, opt)
+	if err != nil {
+		return fmt.Errorf("error listing ssh key -- %v", err)
+	}
+
+	for _, key := range keys {
+		if strings.TrimSpace(key.SSHKey) == strings.TrimSpace(p.SSHPublicKey) {
+			p.SSHKeyID = cast.ToString(key.ID)
+			p.SSHKeyFound = true
+			utils.DebugF("Found SSH Key: %v -- %v ", key.Label, p.SSHKeyID)
+		}
+	}
+
+	// create one if not found
+	if !p.SSHKeyFound {
+		utils.DebugF("No SSHKey found. create a new one")
+		createRequest := linodego.SSHKeyCreateOptions{
+			Label:  p.SSHKeyName,
+			SSHKey: p.SSHPublicKey,
+		}
+		key, err := client.CreateSSHKey(ctx, createRequest)
+		if err == nil {
+			p.SSHKeyID = cast.ToString(key.ID)
+			p.SSHKeyFound = true
+			utils.DebugF("Created new SSH Key: %v", p.SSHKeyID)
+		}
+	}
+
+	return nil
+}
+
+func (p *Provider) ListSnapshotLN() error {
+	client := p.ConvertClientLinode()
+	ctx := context.TODO()
+	opt := &linodego.ListOptions{
+		PageOptions: nil,
+		Filter:      "",
+	}
+
+	snapshots, err := client.ListImages(ctx, opt)
+	if err != nil {
+		return fmt.Errorf("error getting linode images")
+	}
+	for _, image := range snapshots {
+		name := image.Label
+		id := cast.ToString(image.ID)
+		if strings.HasPrefix(name, libs.SNAPSHOT) {
+			p.OldSnapShotID = append(p.OldSnapShotID, id)
+		}
+
+		if strings.TrimSpace(name) == strings.TrimSpace(p.SnapshotName) {
+			utils.DebugF("Found base image snapshot with ID: %s", id)
+			p.SnapshotID = id
+			p.SnapshotName = name
+			p.SnapshotFound = true
+		}
+	}
+
+	return nil
+}
+
+func (p *Provider) LinodeDiskMap() {
+	p.SwapSizeMap = make(map[string]int)
+	p.SwapSizeMap["g6-nanode-1"] = 20000
+	p.SwapSizeMap["g6-standard-1"] = 4000
+	p.SwapSizeMap["g6-standard-2"] = 8000
+	p.SwapSizeMap["g6-standard-4"] = 16000
+	p.SwapSizeMap["g6-standard-6"] = 32000
+}
+
+func (p *Provider) CreateInstanceLN(name string) (dropletId int, err error) {
+	client := p.ConvertClientLinode()
+
+	ctx := context.TODO()
+	booted := false
+	swapSize := 4000
+	if swap, ok := p.SwapSizeMap[p.Size]; ok {
+		swapSize = swap
+	}
+
+	createRequest := linodego.InstanceCreateOptions{
+		Region: p.Region,
+		Type:   p.Size,
+		Label:  name,
+		// RootPass: utils.RandomString(10),
+		RootPass: GeneratePassword(16),
+		AuthorizedKeys: []string{
+			p.SSHPublicKey,
+		},
+		//AuthorizedUsers: []string{"root"},
+		Image: p.SnapshotID,
+		Tags:  []string{libs.SNAPSHOT},
+
+		SwapSize: &swapSize,
+		Booted:   &booted,
+		//StackScriptID:   0,
+		//Group:           "",
+		//StackScriptData: nil,
+		//BackupID:        0,
+		//BackupsEnabled:  false,
+		//PrivateIP:       false,
+	}
+
+	utils.DebugF("Creating instance based on %v/%v image with password: %v", p.Size, p.SnapshotID, createRequest.RootPass)
+	instance, err := client.CreateInstance(ctx, createRequest)
+	if err != nil {
+		utils.ErrorF("error create linode instance %v -- %v", name, err)
+		return dropletId, fmt.Errorf("error create linode instance -- %v", err)
+	}
+
+	//spew.Dump(instance)
+	// get droplet IP info
+	dropletId = instance.ID
+	time.Sleep(60 * time.Second)
+	return dropletId, nil
+}
+
+func (p *Provider) BootInstanceLN(dropletId int) error {
+	client := p.ConvertClientLinode()
+
+	ctx := context.TODO()
+	err := client.BootInstance(ctx, dropletId, 0)
+	return err
+}
+
+func (p *Provider) MountDiskLN(dropletId int) error {
+	client := p.ConvertClientLinode()
+	utils.InforF("Mounting disk: %v", dropletId)
+	ctx := context.TODO()
+
+	disk, err := client.CreateInstanceDisk(ctx, dropletId, linodego.InstanceDiskCreateOptions{
+		Label:      "test1",
+		Filesystem: "ext4",
+		Size:       2000,
+	})
+	if err != nil {
+		utils.ErrorF("Error creating disk for resize: %s", err)
+	}
+
+	disk, err = client.WaitForInstanceDiskStatus(ctx, dropletId, disk.ID, linodego.DiskReady, 180)
+	if err != nil {
+		utils.ErrorF("Error waiting for disk readiness for resize: %s", err)
+		return err
+	}
+	err = client.ResizeInstanceDisk(ctx, dropletId, disk.ID, 4000)
+	if err != nil {
+		utils.ErrorF("Error resizing instance disk: %s", err)
+	}
+
+	return nil
+}
+
+func (p *Provider) InstanceInfoLN(id int) (Instance, error) {
+	var parsedInstance Instance
+	client := p.ConvertClientLinode()
+
+	instance, err := client.GetInstance(context.TODO(), id)
+	if err != nil {
+		return parsedInstance, fmt.Errorf("error getting linode instance")
+	}
+
+	if len(instance.IPv4) == 0 {
+		utils.ErrorF("Instance has no public IP: %v -- %v", instance.ID, instance.Label)
+		return parsedInstance, fmt.Errorf("no public ip address yet")
+	}
+	ipAddress := instance.IPv4[0]
+	parsedInstance = Instance{
+		InstanceID:   cast.ToString(instance.ID),
+		IPAddress:    cast.ToString(ipAddress),
+		InstanceName: instance.Label,
+		ImageID:      cast.ToString(instance.Image),
+		ImageName:    instance.Image,
+		Region:       instance.Region,
+		Memory:       cast.ToString(instance.Specs.Memory),
+		CPU:          cast.ToString(instance.Specs.VCPUs),
+		Disk:         cast.ToString(instance.Specs.Disk),
+		Status:       cast.ToString(instance.Status),
+		CreatedAt:    cast.ToString(instance.Created),
+		InputName:    "",
+		ProviderName: "linode",
+	}
+	p.CreatedInstance = parsedInstance
+	utils.DebugF("Created instance ID: %v -- %v -- %v", p.CreatedInstance.InstanceID, p.CreatedInstance.InstanceName, p.CreatedInstance.IPAddress)
+
+	return parsedInstance, nil
+}
+
+func (p *Provider) DeleteInstanceLN(id string) error {
+	client := p.ConvertClientLinode()
+
+	ctx := context.TODO()
+	err := client.DeleteInstance(ctx, cast.ToInt(id))
+	if err != nil {
+		utils.ErrorF("error delete instance -- %v", err)
+		return fmt.Errorf("error delete instance")
+	}
+	utils.InforF("Deleted instance ID: %v", color.HiRedString(id))
+	return nil
+}
+
+func (p *Provider) DeleteSnapShotLN(id string) error {
+	client := p.ConvertClientLinode()
+
+	ctx := context.TODO()
+	err := client.DeleteImage(ctx, id)
+	if err != nil {
+		utils.ErrorF("error delete snapshot -- %v", err)
+		return fmt.Errorf("error delete instance")
+	}
+	utils.InforF("Deleted snapshot ID: %v", color.HiRedString(id))
+	return nil
+}
+
+// @NOTE: old note for increase disk in linode
+//func (c *CloudRunner) MountDiskLinode() error {
+//	diskSize := p.DisksMap[c.Cloud.Size]
+//	if diskSize == "" {
+//		return fmt.Errorf("can't found disk size")
+//	}
+//
+//	utils.InforF("Mounting Disk size %s for instance %s", diskSize, c.InstanceID)
+//	cmd := c.Prefix + fmt.Sprintf(`linodes disks-list %s`, c.InstanceID)
+//	out := c.RetryCommandWithExcludeString(cmd, `Request failed:`)
+//	jsonParsed, err := gabs.ParseJSON([]byte(out))
+//	if err != nil {
+//		utils.ErrorF("error when parsing content of droplet list")
+//		return err
+//	}
+//
+//	var diskID string
+//	for _, instance := range jsonParsed.Children() {
+//		filesystem := cast.ToString(instance.S("filesystem").Data())
+//		if strings.HasPrefix(filesystem, "ext") {
+//			diskID = cast.ToString(instance.S("id").Data())
+//		}
+//	}
+//	if diskID == "" {
+//		return fmt.Errorf("error to find disk ID")
+//	}
+//
+//	cmd = c.Prefix + fmt.Sprintf(`linodes disk-resize %s %s --size %s`, c.InstanceID, diskID, diskSize)
+//	out = c.RetryCommandWithExcludeString(cmd, `Request failed:`)
+//	if strings.Contains(out, "Request failed") {
+//		return fmt.Errorf("error to mount disk")
+//	}
+//
+//	time.Sleep(5 * time.Second)
+//	// everything done booting the instance up
+//	cmd = c.Prefix + fmt.Sprintf(`linodes boot  %s `, c.InstanceID)
+//	out = c.RetryCommandWithExcludeString(cmd, `Request failed:`)
+//	if strings.Contains(out, "Request failed") {
+//		return fmt.Errorf("error to mount disk")
+//	}
+//
+//	return nil
+//}
+//
+//func (c *CloudRunner) LinodeDiskMap() {
+//	c.DisksMap = make(map[string]string)
+//	c.DisksMap["g6-nanode-1"] = "20000"
+//	c.DisksMap["g6-standard-1"] = "45000"
+//	c.DisksMap["g6-standard-2"] = "75000"
+//	c.DisksMap["g6-standard-4"] = "150000"
+//	c.DisksMap["g6-standard-6"] = "320000"
+//}
+//
+//var reservedAddrRanges []*net.IPNet
+//
+//var ReservedCIDRs = []string{
+//	"192.168.0.0/16",
+//	"172.16.0.0/12",
+//	"10.0.0.0/8",
+//	"127.0.0.0/8",
+//	"224.0.0.0/4",
+//	"240.0.0.0/4",
+//	"100.64.0.0/10",
+//	"198.18.0.0/15",
+//	"169.254.0.0/16",
+//	"192.88.99.0/24",
+//	"192.0.0.0/24",
+//	"192.0.2.0/24",
+//	"192.94.77.0/24",
+//	"192.94.78.0/24",
+//	"192.52.193.0/24",
+//	"192.12.109.0/24",
+//	"192.31.196.0/24",
+//	"192.0.0.0/29",
+//}
+//
+//func init() {
+//	for _, cidr := range ReservedCIDRs {
+//		if _, ipnet, err := net.ParseCIDR(cidr); err == nil {
+//			reservedAddrRanges = append(reservedAddrRanges, ipnet)
+//		}
+//	}
+//}
+//
+//// IsPrivateIP checks if the addr parameter is within one of the address ranges in the ReservedCIDRs slice.
+//func IsPrivateIP(addr string) bool {
+//	ip := net.ParseIP(addr)
+//	if ip == nil {
+//		return false
+//	}
+//
+//	var cidr string
+//	for _, block := range reservedAddrRanges {
+//		if block.Contains(ip) {
+//			cidr = block.String()
+//			break
+//		}
+//	}
+//
+//	if cidr != "" {
+//		return true
+//	}
+//	return false
+//}

provider/provider_test.go

@@ -0,0 +1,95 @@
+package provider
+
+import (
+	"testing"
+
+	"github.com/davecgh/go-spew/spew"
+)
+
+var sampleToken = "d17ab13b0a3fccafefc932b9db95be45464339073100423336045977c0924491"
+
+func TestProviderList(t *testing.T) {
+	provider, err := InitProvider("do", sampleToken)
+	if err != nil {
+		t.Errorf("error")
+	}
+	provider.SSHPublicKey = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC0ZmMumN5GTKuPXVVqugz+6BZs6JyNaQGsPgsZk86uON//oXi5fkutsNu5IIPDZXph3P5NbUj2dPalNyzNY5jClPJeT0F/eowIhPeo5GfjmJlXR4TTgHSwPYrOQNop2w+xh2z5h4IXMEPVLpEsN67MuUjTKzRwirwPjigZS/gayhwfOfDsPaIwBhpoBGuR1x+Rzzxuiy7TToNoWhF6pT9qONoCtr0VrPMsmjVpEPKD/uTW/8KeFL0pb/9z18M4IlbtvkO0Y6RhrpFGNSmZTWc1eDsJpFJerrVd48rgx3aRHriijl4zX4GBhc0zjqJwv+nGTGFPJ9Tx/3kPMDUGna/f91VU7sL7YqeiSed8S0YcWfntYy64OknvMpN8VIoQ7WiJAkR3wPw+tL3ZduXXAiKHFTAiXev02mOvo2F2nQKdGS98lOH5m+zuUm8abYbyXYlGNEzz576ksb6nMWCSSXwhA5f4clPKaPmgBQFQMUtq6Wgb8Fjq2r1MpjIWwUvx84s= osmp-cloud"
+	provider.Prepare()
+
+	provider.ListInstance()
+	spew.Dump(provider.Instances)
+}
+
+func TestProviderCreate(t *testing.T) {
+	provider, err := InitProvider("do", sampleToken)
+	if err != nil {
+		t.Errorf("error ")
+	}
+	provider.SSHPublicKey = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC0ZmMumN5GTKuPXVVqugz+6BZs6JyNaQGsPgsZk86uON//oXi5fkutsNu5IIPDZXph3P5NbUj2dPalNyzNY5jClPJeT0F/eowIhPeo5GfjmJlXR4TTgHSwPYrOQNop2w+xh2z5h4IXMEPVLpEsN67MuUjTKzRwirwPjigZS/gayhwfOfDsPaIwBhpoBGuR1x+Rzzxuiy7TToNoWhF6pT9qONoCtr0VrPMsmjVpEPKD/uTW/8KeFL0pb/9z18M4IlbtvkO0Y6RhrpFGNSmZTWc1eDsJpFJerrVd48rgx3aRHriijl4zX4GBhc0zjqJwv+nGTGFPJ9Tx/3kPMDUGna/f91VU7sL7YqeiSed8S0YcWfntYy64OknvMpN8VIoQ7WiJAkR3wPw+tL3ZduXXAiKHFTAiXev02mOvo2F2nQKdGS98lOH5m+zuUm8abYbyXYlGNEzz576ksb6nMWCSSXwhA5f4clPKaPmgBQFQMUtq6Wgb8Fjq2r1MpjIWwUvx84s= osmp-cloud"
+	provider.Prepare()
+
+	id, err := provider.CreateInstanceDO("example.com")
+	if err == nil {
+		provider.GetInstanceInfo(id)
+	}
+	spew.Dump(provider.Instances)
+}
+
+func TestProviderDelete(t *testing.T) {
+	provider, err := InitProvider("do", sampleToken)
+	if err != nil {
+		t.Errorf("error ")
+	}
+
+	provider.DeleteInstance("258443728")
+	spew.Dump(provider.Instances)
+}
+
+func TestProviderAccount(t *testing.T) {
+	provider, err := InitProvider("do", sampleToken)
+	if err != nil {
+		t.Errorf("error ")
+	}
+
+	provider.AccountDO()
+}
+
+func TestProvider_DeleteSnapshot(t *testing.T) {
+	provider, err := InitProvider("do", sampleToken)
+	if err != nil {
+		t.Errorf("error ")
+	}
+
+	provider.DeleteSnapShotDO("89053110")
+	provider.ListSnapshotDO()
+	spew.Dump(provider.OldSnapShotID)
+	spew.Dump(provider.SnapshotID)
+}
+
+func TestProviderLN(t *testing.T) {
+	sampleToken = "6634197757df67b24753d0d241003ae09afd53bc7f9648c191e37acb61bfef37"
+	provider, err := InitProvider("linode", sampleToken)
+	if err != nil {
+		t.Errorf("error ")
+	}
+	provider.SSHPublicKey = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC0ZmMumN5GTKuPXVVqugz+6BZs6JyNaQGsPgsZk86uON//oXi5fkutsNu5IIPDZXph3P5NbUj2dPalNyzNY5jClPJeT0F/eowIhPeo5GfjmJlXR4TTgHSwPYrOQNop2w+xh2z5h4IXMEPVLpEsN67MuUjTKzRwirwPjigZS/gayhwfOfDsPaIwBhpoBGuR1x+Rzzxuiy7TToNoWhF6pT9qONoCtr0VrPMsmjVpEPKD/uTW/8KeFL0pb/9z18M4IlbtvkO0Y6RhrpFGNSmZTWc1eDsJpFJerrVd48rgx3aRHriijl4zX4GBhc0zjqJwv+nGTGFPJ9Tx/3kPMDUGna/f91VU7sL7YqeiSed8S0YcWfntYy64OknvMpN8VIoQ7WiJAkR3wPw+tL3ZduXXAiKHFTAiXev02mOvo2F2nQKdGS98lOH5m+zuUm8abYbyXYlGNEzz576ksb6nMWCSSXwhA5f4clPKaPmgBQFQMUtq6Wgb8Fjq2r1MpjIWwUvx84s= osmp-cloud"
+	//provider.GetSSHKey()
+
+	provider.Prepare()
+	//id, _ := provider.CreateInstanceLN("new2")
+	//time.Sleep(90*time.Second)
+	//id := 29530061
+	//provider.MountDiskLN(id)
+	provider.DeleteInstance("29530163")
+	//provider.AccountLN()
+	spew.Dump(provider.Instances)
+}
+
+func TestProviderAccountAWS(t *testing.T) {
+	provider, err := InitProvider("aws", sampleToken)
+	if err != nil {
+		t.Errorf("error ")
+	}
+
+	provider.AccountDO()
+}

server/auth.go

@@ -0,0 +1,68 @@
+package server
+
+import (
+	"time"
+
+	// jwtware "github.com/gofiber/jwt/v2"
+
+	"github.com/dgrijalva/jwt-go"
+	"github.com/gofiber/fiber/v2"
+)
+
+// authen stuff
+
+func Login(c *fiber.Ctx) error {
+	type LoginInput struct {
+		Username string `json:"username"`
+		Password string `json:"password"`
+	}
+	var input LoginInput
+	if err := c.BodyParser(&input); err != nil {
+		return c.SendStatus(fiber.StatusUnauthorized)
+	}
+	user := input.Username
+	pass := input.Password
+
+	dbUser := Opt.Client.Username
+	dbPass := Opt.Client.Password
+
+	// auto pass if -A is specify
+	if Opt.Server.NoAuthen {
+		user = dbUser
+		pass = dbPass
+	}
+
+	// Throws Unauthorized error
+	if dbUser != user || dbPass != pass {
+		return c.SendStatus(fiber.StatusUnauthorized)
+	}
+
+	// Create the Claims
+	claims := jwt.MapClaims{
+		"name":  "Osmdeus Default",
+		"admin": true,
+		"exp":   time.Now().Add(time.Hour * 24 * 30).Unix(),
+	}
+
+	// Create token
+	token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims)
+
+	// Generate encoded token and send it as response.
+	t, err := token.SignedString([]byte(Opt.Server.JWTSecret))
+	if err != nil {
+		return c.SendStatus(fiber.StatusInternalServerError)
+	}
+	return c.JSON(fiber.Map{"status": "success", "message": "Successfully login", "token": t})
+
+}
+
+func jwtError(c *fiber.Ctx, err error) error {
+	if err.Error() == "Missing or malformed JWT" {
+		c.Status(fiber.StatusBadRequest)
+		return c.JSON(fiber.Map{"status": "error", "message": "Missing or malformed JWT", "data": nil})
+
+	} else {
+		c.Status(fiber.StatusUnauthorized)
+		return c.JSON(fiber.Map{"status": "error", "message": "Invalid or expired JWT", "data": nil})
+	}
+}

server/builder.go

@@ -0,0 +1,170 @@
+package server
+
+import (
+	"fmt"
+	"github.com/gofiber/fiber/v2"
+	"github.com/j3ssie/osmedeus/libs"
+	"github.com/j3ssie/osmedeus/utils"
+	"os"
+	"path"
+	"strings"
+)
+
+// UploadData data required in json form
+type UploadData struct {
+	Data     string `json:"data"`
+	Filename string `json:"filename"`
+}
+
+// Upload testing authenticated connection
+func Upload(c *fiber.Ctx) error {
+	var uploadData UploadData
+
+	err := c.BodyParser(&uploadData)
+	if err != nil {
+		return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
+			"error": "Cannot parse JSON",
+		})
+	}
+
+	if uploadData.Filename == "" {
+		uploadData.Filename = utils.RandomString(6)
+	}
+	tmpFile := path.Base(utils.NormalizePath(uploadData.Filename))
+	baseDir := fmt.Sprintf("/tmp/%v-input/", libs.BINARY)
+	if !utils.FolderExists(baseDir) {
+		os.MkdirAll(baseDir, 0755)
+	}
+
+	filename := path.Join(baseDir, tmpFile)
+	data := uploadData.Data
+
+	utils.WriteToFile(filename, data)
+
+	return c.JSON(ResponseHTTP{
+		Status: 200,
+		Data: fiber.Map{
+			"filepath": filename,
+		},
+		Type:    "upload",
+		Message: "New Data Uploaded",
+	})
+
+}
+
+// SaveTargets save upload data to /tmp/osm-input/data-osm-xxx.txt
+func SaveTargets(targets []string) string {
+	tmpFile := fmt.Sprintf("data-%v-%v-%v.txt", utils.GetTS(), libs.BINARY, utils.RandomString(6))
+	baseDir := fmt.Sprintf("/tmp/%v-input/", libs.BINARY)
+	if !utils.FolderExists(baseDir) {
+		os.MkdirAll(baseDir, 0755)
+	}
+	targetFile := path.Join(baseDir, tmpFile)
+	data := strings.Join(targets, "\n")
+	filename, err := utils.WriteToFile(targetFile, data)
+	if err != nil {
+		return ""
+	}
+	return filename
+}
+
+// CommandBuilder build core command from API
+func CommandBuilder(taskData *TaskData) string {
+	binary := fmt.Sprintf("%s scan", libs.BINARY)
+	if taskData.Distributed {
+		binary = fmt.Sprintf("%s cloud", libs.BINARY)
+		if taskData.Chunk {
+			binary = fmt.Sprintf("%s cloud --chunk", libs.BINARY)
+		}
+	}
+
+	if taskData.ScanID != "" {
+		binary += fmt.Sprintf(" --sid %v ", taskData.ScanID)
+	}
+
+	var command string
+	var workspace, concurrency, timeout, params, workflow, plugin, scanID string
+
+	if len(taskData.TargetsList) > 0 {
+		taskData.TargetsFile = SaveTargets(taskData.TargetsList)
+		utils.DebugF("Save targets list to: %v", taskData.TargetsFile)
+	}
+
+	// get workspace
+	if taskData.Workspace != "" {
+		taskData.Workspace = utils.CleanPath(taskData.Workspace)
+		workspace = fmt.Sprintf(" -w '%v'", taskData.Workspace)
+	}
+
+	if taskData.Binary != "" {
+		binary = taskData.Binary
+	}
+
+	//if taskData.RawName {
+	//	binary = binary + " --rt "
+	//}
+
+	if taskData.ViewOnly {
+		binary = binary + " --view-only "
+	}
+
+	// default workflow is general
+	if taskData.WorkFlow != "" {
+		workflow = fmt.Sprintf(" -f '%v'", taskData.WorkFlow)
+	}
+
+	// some mics options
+	if taskData.Timeout != "" {
+		timeout = fmt.Sprintf(" --timeout '%v'", taskData.Timeout)
+	}
+	if taskData.Concurrency > 0 {
+		concurrency = fmt.Sprintf(" -c %v", taskData.Concurrency)
+	}
+	if len(taskData.Params) > 0 {
+		for _, param := range taskData.Params {
+			// @NOTE replace ',' from request to ';;' first because corbra auto split ','
+			if strings.Contains(param, ",") {
+				param = strings.Replace(param, ",", ";;", -1)
+			}
+
+			if strings.HasPrefix(param, "'") && strings.HasSuffix(param, "'") {
+				params += fmt.Sprintf(" -p %s", param)
+				continue
+			}
+			params += fmt.Sprintf(" -p '%s'", param)
+		}
+	}
+
+	if taskData.PluginName != "" {
+		plugin = fmt.Sprintf(" -m '%v'", taskData.PluginName)
+	}
+
+	// override everything
+	if taskData.Command != "" {
+		return taskData.Command
+	}
+
+	// mean general scan
+	if taskData.PluginName == "" {
+		command = fmt.Sprintf("%v %v -t %v %v%v%v%v", binary, workflow, taskData.Target, concurrency, timeout, workspace, params)
+		if taskData.TargetsFile != "" {
+			command = fmt.Sprintf("%v %v -T %v %v%v%v%v", binary, workflow, taskData.TargetsFile, concurrency, timeout, workspace, params)
+		}
+		command = strings.TrimSpace(command)
+		if taskData.Debug {
+			command = command + " --debug"
+		}
+		return command
+	}
+
+	command = fmt.Sprintf("%v %v -t %v %v %v%v%v%v", binary, plugin, taskData.Target, scanID, concurrency, timeout, workspace, params)
+	if taskData.TargetsFile != "" {
+		command = fmt.Sprintf("%v %v -t %v %v %v%v%v%v", binary, plugin, taskData.TargetsFile, scanID, concurrency, timeout, workspace, params)
+	}
+
+	command = strings.TrimSpace(command)
+	if taskData.Debug {
+		command = command + " --debug"
+	}
+	return command
+}

server/docs/docs.go

@@ -0,0 +1,120 @@
+// GENERATED BY THE COMMAND ABOVE; DO NOT EDIT
+// This file was generated by swaggo/swag
+
+package docs
+
+import (
+	"bytes"
+	"encoding/json"
+	"strings"
+
+	"github.com/alecthomas/template"
+	"github.com/swaggo/swag"
+)
+
+var doc = `{
+    "schemes": {{ marshal .Schemes }},
+    "swagger": "2.0",
+    "info": {
+        "description": "{{.Description}}",
+        "title": "{{.Title}}",
+        "contact": {},
+        "version": "{{.Version}}"
+    },
+    "host": "{{.Host}}",
+    "basePath": "{{.BasePath}}",
+    "paths": {
+        "/v1/books": {
+            "get": {
+                "description": "Get all books",
+                "consumes": [
+                    "application/json"
+                ],
+                "produces": [
+                    "application/json"
+                ],
+                "tags": [
+                    "books"
+                ],
+                "summary": "Get all books",
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "$ref": "#/definitions/handler.ResponseHTTP"
+                        }
+                    },
+                    "503": {
+                        "description": "Service Unavailable",
+                        "schema": {
+                            "$ref": "#/definitions/handler.ResponseHTTP"
+                        }
+                    }
+                }
+            }
+        }
+    },
+    "definitions": {
+        "handler.ResponseHTTP": {
+            "type": "object",
+            "properties": {
+                "data": {
+                    "type": "object"
+                },
+                "message": {
+                    "type": "string"
+                },
+                "success": {
+                    "type": "boolean"
+                }
+            }
+        }
+    }
+}`
+
+type swaggerInfo struct {
+	Version     string
+	Host        string
+	BasePath    string
+	Schemes     []string
+	Title       string
+	Description string
+}
+
+// SwaggerInfo holds exported Swagger Info so clients can modify it
+var SwaggerInfo = swaggerInfo{
+	Version:     "",
+	Host:        "",
+	BasePath:    "",
+	Schemes:     []string{},
+	Title:       "",
+	Description: "",
+}
+
+type s struct{}
+
+func (s *s) ReadDoc() string {
+	sInfo := SwaggerInfo
+	sInfo.Description = strings.Replace(sInfo.Description, "\n", "\\n", -1)
+
+	t, err := template.New("swagger_info").Funcs(template.FuncMap{
+		"marshal": func(v interface{}) string {
+			a, _ := json.Marshal(v)
+			return string(a)
+		},
+	}).Parse(doc)
+	if err != nil {
+		return doc
+	}
+
+	var tpl bytes.Buffer
+	if err := t.Execute(&tpl, sInfo); err != nil {
+		return doc
+	}
+
+	return tpl.String()
+}
+
+func init() {
+	swag.Register(swag.Name, &s{})
+}