From 89a7d36df32d7c984611c2b0fef6c6fba9e81df2 Mon Sep 17 00:00:00 2001 From: j3ssie Date: Thu, 7 Sep 2023 21:21:14 +0700 Subject: [PATCH] Release v4.6.0 --- .github/FUNDING.yml | 12 + .gitignore | 5 + LICENSE | 22 + Makefile | 38 + README.md | 157 +++ cmd/cloud.go | 185 ++++ cmd/config.go | 100 ++ cmd/exec.go | 56 ++ cmd/health.go | 283 ++++++ cmd/provider.go | 266 ++++++ cmd/queue.go | 87 ++ cmd/report.go | 162 ++++ cmd/root.go | 147 +++ cmd/scan.go | 80 ++ cmd/server.go | 49 + cmd/update.go | 77 ++ cmd/usage.go | 313 ++++++ cmd/utils.go | 139 +++ cmd/version.go | 176 ++++ cmd/workflow.go | 200 ++++ core/backup.go | 68 ++ core/banner.go | 63 ++ core/config.go | 382 ++++++++ core/cron.go | 30 + core/db.go | 229 +++++ core/external.go | 298 ++++++ core/flow.go | 222 +++++ core/import.go | 574 +++++++++++ core/import_mics.go | 238 +++++ core/markdown.go | 256 +++++ core/mode_test.go | 24 + core/module.go | 370 ++++++++ core/parse.go | 322 +++++++ core/parse_test.go | 71 ++ core/queue.go | 147 +++ core/reference.go | 104 ++ core/report.go | 215 +++++ core/runner.go | 373 ++++++++ core/runtime.go | 488 ++++++++++ core/step.go | 167 ++++ core/tmux.go | 85 ++ core/token.go | 288 ++++++ core/update.go | 336 +++++++ core/validate.go | 144 +++ core/validate_test.go | 53 ++ database/connect.go | 63 ++ database/models.go | 99 ++ database/select.go | 100 ++ distribute/clean.go | 54 ++ distribute/cloud_runner.go | 201 ++++ distribute/command.go | 175 ++++ distribute/create.go | 153 +++ distribute/db_cloud.go | 35 + distribute/health.go | 159 ++++ distribute/mics.go | 115 +++ distribute/routine.go | 94 ++ distribute/scan.go | 267 ++++++ distribute/ssh.go | 430 +++++++++ distribute/wizard.go | 240 +++++ execution/clean.go | 527 ++++++++++ execution/clean_test.go | 19 + execution/git.go | 183 ++++ execution/git_test.go | 25 + execution/gitlab.go | 206 ++++ execution/gitlab_test.go | 9 + execution/noti.go | 352 +++++++ execution/noti_test.go | 44 + execution/process.go | 93 ++ execution/process_test.go | 9 + execution/remote.go | 126 +++ execution/request.go | 227 +++++ execution/require.go | 106 +++ execution/s3_cdn.go | 125 +++ execution/scripts.go | 225 +++++ execution/scripts_test.go | 19 + execution/split.go | 126 +++ execution/wildcard.go | 61 ++ go.mod | 113 +++ go.sum | 737 ++++++++++++++ libs/cloud.go | 115 +++ libs/flow.go | 69 ++ libs/mics.go | 61 ++ libs/noti.go | 26 + libs/options.go | 172 ++++ libs/queue.go | 22 + libs/step.go | 30 + libs/update.go | 28 + libs/version.go | 26 + main.go | 7 + provider/action.go | 201 ++++ provider/building.go | 157 +++ provider/parser.go | 127 +++ provider/provider.go | 265 ++++++ provider/provider_aws.go | 516 ++++++++++ provider/provider_digitalocean.go | 294 ++++++ provider/provider_linode.go | 477 ++++++++++ provider/provider_test.go | 95 ++ server/auth.go | 68 ++ server/builder.go | 170 ++++ server/docs/docs.go | 120 +++ server/docs/swagger.json | 53 ++ server/docs/swagger.yaml | 33 + server/mics.go | 100 ++ server/ping.go | 41 + server/router.go | 152 +++ server/scan.go | 130 +++ server/ssl.go | 191 ++++ server/workspace.go | 118 +++ test-workflows/flow-with-params.yaml | 11 + test-workflows/general.yaml | 31 + test-workflows/general/archive.yaml | 10 + test-workflows/general/cloudbrute.yaml | 10 + test-workflows/general/credintel.yaml | 10 + test-workflows/general/dirbscan.yaml | 10 + test-workflows/general/fingerprint.yaml | 13 + test-workflows/general/ipspace.yaml | 10 + test-workflows/general/portscan.yaml | 10 + test-workflows/general/probing.yaml | 10 + test-workflows/general/spider.yaml | 10 + test-workflows/general/subdomain.yaml | 10 + test-workflows/general/summary.yaml | 10 + test-workflows/general/vulnscan.yaml | 11 + test-workflows/parallel.yaml | 14 + test-workflows/pre-run-cloud.yaml | 17 + test-workflows/sample/parallel.yaml | 20 + test-workflows/sample/parallel2.yaml | 12 + test-workflows/sample/timeout-module.yaml | 6 + test-workflows/serial.yaml | 12 + test-workflows/test-module/loop-step.yaml | 20 + .../test-module/markdown-generate.yaml | 15 + test-workflows/test-module/ose.yaml | 20 + test-workflows/test-module/s3-cdn.yaml | 8 + test-workflows/test-module/set-var.yaml | 9 + test-workflows/test-module/test-parallel.yaml | 18 + .../test-module/with-threadshold.yaml | 14 + utils/helper.go | 897 ++++++++++++++++++ utils/log.go | 128 +++ utils/request.go | 122 +++ 138 files changed, 18980 insertions(+) create mode 100644 .github/FUNDING.yml create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 README.md create mode 100644 cmd/cloud.go create mode 100644 cmd/config.go create mode 100644 cmd/exec.go create mode 100644 cmd/health.go create mode 100644 cmd/provider.go create mode 100644 cmd/queue.go create mode 100644 cmd/report.go create mode 100644 cmd/root.go create mode 100644 cmd/scan.go create mode 100644 cmd/server.go create mode 100644 cmd/update.go create mode 100644 cmd/usage.go create mode 100644 cmd/utils.go create mode 100644 cmd/version.go create mode 100644 cmd/workflow.go create mode 100644 core/backup.go create mode 100644 core/banner.go create mode 100644 core/config.go create mode 100644 core/cron.go create mode 100644 core/db.go create mode 100644 core/external.go create mode 100644 core/flow.go create mode 100644 core/import.go create mode 100644 core/import_mics.go create mode 100644 core/markdown.go create mode 100644 core/mode_test.go create mode 100644 core/module.go create mode 100644 core/parse.go create mode 100644 core/parse_test.go create mode 100644 core/queue.go create mode 100644 core/reference.go create mode 100644 core/report.go create mode 100644 core/runner.go create mode 100644 core/runtime.go create mode 100644 core/step.go create mode 100644 core/tmux.go create mode 100644 core/token.go create mode 100644 core/update.go create mode 100644 core/validate.go create mode 100644 core/validate_test.go create mode 100644 database/connect.go create mode 100644 database/models.go create mode 100644 database/select.go create mode 100644 distribute/clean.go create mode 100644 distribute/cloud_runner.go create mode 100644 distribute/command.go create mode 100644 distribute/create.go create mode 100644 distribute/db_cloud.go create mode 100644 distribute/health.go create mode 100644 distribute/mics.go create mode 100644 distribute/routine.go create mode 100644 distribute/scan.go create mode 100644 distribute/ssh.go create mode 100644 distribute/wizard.go create mode 100644 execution/clean.go create mode 100644 execution/clean_test.go create mode 100644 execution/git.go create mode 100644 execution/git_test.go create mode 100644 execution/gitlab.go create mode 100644 execution/gitlab_test.go create mode 100644 execution/noti.go create mode 100644 execution/noti_test.go create mode 100644 execution/process.go create mode 100644 execution/process_test.go create mode 100644 execution/remote.go create mode 100644 execution/request.go create mode 100644 execution/require.go create mode 100644 execution/s3_cdn.go create mode 100644 execution/scripts.go create mode 100644 execution/scripts_test.go create mode 100644 execution/split.go create mode 100644 execution/wildcard.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 libs/cloud.go create mode 100644 libs/flow.go create mode 100644 libs/mics.go create mode 100644 libs/noti.go create mode 100644 libs/options.go create mode 100644 libs/queue.go create mode 100644 libs/step.go create mode 100644 libs/update.go create mode 100644 libs/version.go create mode 100644 main.go create mode 100644 provider/action.go create mode 100644 provider/building.go create mode 100644 provider/parser.go create mode 100644 provider/provider.go create mode 100644 provider/provider_aws.go create mode 100644 provider/provider_digitalocean.go create mode 100644 provider/provider_linode.go create mode 100644 provider/provider_test.go create mode 100644 server/auth.go create mode 100644 server/builder.go create mode 100644 server/docs/docs.go create mode 100644 server/docs/swagger.json create mode 100644 server/docs/swagger.yaml create mode 100644 server/mics.go create mode 100644 server/ping.go create mode 100644 server/router.go create mode 100644 server/scan.go create mode 100644 server/ssl.go create mode 100644 server/workspace.go create mode 100644 test-workflows/flow-with-params.yaml create mode 100644 test-workflows/general.yaml create mode 100644 test-workflows/general/archive.yaml create mode 100644 test-workflows/general/cloudbrute.yaml create mode 100644 test-workflows/general/credintel.yaml create mode 100644 test-workflows/general/dirbscan.yaml create mode 100644 test-workflows/general/fingerprint.yaml create mode 100644 test-workflows/general/ipspace.yaml create mode 100644 test-workflows/general/portscan.yaml create mode 100644 test-workflows/general/probing.yaml create mode 100644 test-workflows/general/spider.yaml create mode 100644 test-workflows/general/subdomain.yaml create mode 100644 test-workflows/general/summary.yaml create mode 100644 test-workflows/general/vulnscan.yaml create mode 100644 test-workflows/parallel.yaml create mode 100644 test-workflows/pre-run-cloud.yaml create mode 100644 test-workflows/sample/parallel.yaml create mode 100644 test-workflows/sample/parallel2.yaml create mode 100644 test-workflows/sample/timeout-module.yaml create mode 100644 test-workflows/serial.yaml create mode 100644 test-workflows/test-module/loop-step.yaml create mode 100644 test-workflows/test-module/markdown-generate.yaml create mode 100644 test-workflows/test-module/ose.yaml create mode 100644 test-workflows/test-module/s3-cdn.yaml create mode 100644 test-workflows/test-module/set-var.yaml create mode 100644 test-workflows/test-module/test-parallel.yaml create mode 100644 test-workflows/test-module/with-threadshold.yaml create mode 100644 utils/helper.go create mode 100644 utils/log.go create mode 100644 utils/request.go diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..ccf10a2 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,12 @@ +# These are supported funding model platforms + +github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] +patreon: j3ssie +open_collective: osmedeus +ko_fi: j3ssie +tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel +community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry +liberapay: # Replace with a single Liberapay username +issuehunt: # Replace with a single IssueHunt username +otechie: # Replace with a single Otechie username +custom: [ 'https://paypal.me/j3ssiejjj' ] \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9571790 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +.DS_STORE +.idea +out* +osmedeus +dist/* diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..c69c3a0 --- /dev/null +++ b/LICENSE @@ -0,0 +1,22 @@ +MIT License + +Copyright (c) 2020 j3ssie + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..48226d9 --- /dev/null +++ b/Makefile @@ -0,0 +1,38 @@ +TARGET ?= osmedeus +GO ?= go +GOFLAGS ?= +VERSION := $(shell cat libs/version.go | grep 'VERSION =' | cut -d '"' -f 2) + +build: + go install + go build -ldflags="-s -w" -tags netgo -trimpath -buildmode=pie -o dist/$(TARGET) + +release: + go install + @echo "==> Clean up old builds" + rm -rf ./dist/* ~/myGit/premium-$(TARGET)-base/dist/* ~/org-$(TARGET)/$(TARGET)-base/dist/* + @echo "==> building binaries for for mac intel" + GOOS=darwin GOARCH=amd64 go build -ldflags="-s -w" -tags netgo -trimpath -buildmode=pie -o dist/$(TARGET) + zip -9 -j dist/$(TARGET)-macos-amd64.zip dist/$(TARGET) && rm -rf ./dist/$(TARGET) + @echo "==> building binaries for for mac M1 chip" + CGO_ENABLED=1 GOOS=darwin GOARCH=arm64 go build -ldflags="-s -w" -tags netgo -trimpath -buildmode=pie -o dist/$(TARGET) + zip -9 -j dist/$(TARGET)-macos-arm64.zip dist/$(TARGET)&& rm -rf ./dist/$(TARGET) + @echo "==> building binaries for linux intel build on mac" + GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -tags netgo -trimpath -buildmode=pie -o dist/$(TARGET) + zip -j dist/$(TARGET)-linux.zip dist/$(TARGET)&& rm -rf ./dist/$(TARGET) + cp dist/* ~/myGit/premium-$(TARGET)-base/dist/ + cp dist/* ~/org-$(TARGET)/$(TARGET)-base/dist/ + @echo "==> Generating metadata info" + $(TARGET) update --gen dist/public.json + mv dist/$(TARGET)-macos-amd64.zip dist/$(TARGET)-$(VERSION)-macos-amd64.zip + mv dist/$(TARGET)-macos-arm64.zip dist/$(TARGET)-$(VERSION)-macos-arm64.zip + mv dist/$(TARGET)-linux.zip dist/$(TARGET)-$(VERSION)-linux.zip +run: + $(GO) $(GOFLAGS) run *.go + +fmt: + $(GO) $(GOFLAGS) fmt ./...; \ + echo "Done." + +test: + $(GO) $(GOFLAGS) test ./... -v% \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..559aa9d --- /dev/null +++ b/README.md @@ -0,0 +1,157 @@ +# Osmedeus Core Engine + +

+ Osmedeus +
+ Osmedeus - A Workflow Engine for Offensive Security + +

+ + + + + +

+

+ +*** + +## šŸ”„ What is Osmedeus? + +Osmedeus is a Workflow Engine for Offensive Security. It was designed to build a foundation with the capability and +flexibility that allows you to build your own reconnaissance system and run it on a large number of targets. + +## šŸ“– Documentation & FAQ + +You can check out the documentation at [**docs.osmedeus.org**](https://docs.osmedeus.org) and the Frequently Asked +Questions at [**here**](https://docs.osmedeus.org/faq) for more information. + +## šŸ“¦ Installation + +> NOTE that you need some essential tools like `curl, wget, git, zip` and login as **root** to start + +```bash +bash <(curl -fsSL https://raw.githubusercontent.com/osmedeus/osmedeus-base/master/install.sh) +``` + +### Build the engine from the source + +Make sure you installed `golang >= v1.17` + +```bash +go install -v github.com/j3ssie/osmedeus@latest +``` + +Check out [**this page**](https://docs.osmedeus.org/installation/) for more the install on other platforms and [**docker +image**](https://docs.osmedeus.org/installation/using-docker/). + +## šŸš€ Key Features of Osmedeus + +- [x] Significantly speed up your recon process +- [x] Organize your scan results +- [x] Efficiently to customize and optimize your recon process +- [x] Seamlessly integrate with new public and private tools +- [x] Easy to scale across large number of targets +- [x] Easy to synchronize the results across many places + +## šŸ’” Usage + +```bash +# Example Scan Commands: + ## Start a simple scan with default 'general' flow + osmedeus scan -t sample.com + + ## Start a general scan but exclude some of the module + osmedeus scan -t sample.com -x screenshot -x spider + + ## Start a scan directly with a module with inputs as a list of http domains like this https://sub.example.com + osmedeus scan -m content-discovery -t http-file.txt + + ## Initiate the scan using a speed option other than the default setting + osmedeus scan -f vuln --tactic gently -t sample.com + osmedeus scan --threads-hold=10 -t sample.com + osmedeus scan -B 5 -t sample.com + + ## Start a simple scan with other flow + osmedeus scan -f vuln -t sample.com + osmedeus scan -f extensive -t sample.com -t another.com + osmedeus scan -f urls -t list-of-urls.txt + + ## Scan list of targets + osmedeus scan -T list_of_targets.txt + osmedeus scan -f vuln -T list-of-targets.txt + + ## Performing static vulnerability scan and secret scan on a git repo + osmedeus scan -m repo-scan -t https://github.com/j3ssie/sample-repo + osmedeus scan -m repo-scan -t /tmp/source-code-folder + osmedeus scan -m repo-scan -T list-of-repo.txt + + ## Scan for CIDR with file contains CIDR with the format '1.2.3.4/24' + osmedeus scan -f cidr -t list-of-ciders.txt + osmedeus scan -f cidr -t '1.2.3.4/24' # this will auto convert the single input to the file and run + + ## Directly run on vuln scan and directory scan on list of domains + osmedeus scan -f domains -t list-of-domains.txt + osmedeus scan -f vuln-and-dirb -t list-of-domains.txt + + ## Use a custom wordlist + osmedeus scan -t sample.com -p 'wordlists={{Data}}/wordlists/content/big.txt' + + ## Use a custom wordlist + cat list_of_targets.txt | osmedeus scan -c 2 + + ## Start a normal scan and backup entire workflow folder to the backup folder + osmedeus scan --backup -f domains -t list-of-subdomains.txt + + ## Start the scan with chunk inputs to review the output way more much faster + osmedeus scan --chunk --chunk-parts 20 -f cidr -t list-of-100-cidr.txt + + ## Continuously run the scan on a target right after it finished + osmedeus utils cron --for --cmd 'osmedeus scan -t example.com' + + ## Backing up all workspaces + ls ~/workspaces-osmedeus | osmedeus report compress + + +# Scan Usage: + osmedeus scan -f [flowName] -t [target] + osmedeus scan -m [modulePath] -T [targetsFile] + osmedeus scan -f /path/to/flow.yaml -t [target] + osmedeus scan -m /path/to/module.yaml -t [target] --params 'port=9200' + osmedeus scan -m /path/to/module.yaml -t [target] -l /tmp/log.log + osmedeus scan --tactic aggressive -m module -t [target] + cat targets | osmedeus scan -f sample + +# Practical Scan Usage: + osmedeus scan -T list_of_targets.txt -W custom_workspaces + osmedeus scan -t target.com -w workspace_name --debug + osmedeus scan -f general -t sample.com + osmedeus scan --tactic aggressive -f general -t sample.com + osmedeus scan -f extensive -t sample.com -t another.com + cat list_of_urls.txt | osmedeus scan -f urls + osmedeus scan --threads-hold=15 -f cidr -t 1.2.3.4/24 + osmedeus scan -m ~/.osmedeus/core/workflow/test/dirbscan.yaml -t list_of_urls.txt + osmedeus scan --wfFolder ~/custom-workflow/ -f your-custom-workflow -t list_of_urls.txt + osmedeus scan --chunk --chunk-part 40 -c 2 -f cidr -t list-of-cidr.txt + +šŸ’” For full help message, please run: osmedeus --hh or osmedeus scan --hh +šŸ“– Documentation can be found here: https://docs.osmedeus.org +``` + +Check out [**this page**](https://docs.osmedeus.org/installation/usage/) for full usage and the [**Practical Usage**](https://docs.osmedeus.org/installation/practical-usage/) to see how to use Osmedeus in a practical way. + +## šŸ’¬ Community & Discussion + +Join Our Discord server [here](https://discord.gg/mtQG2FQsYA) + +## šŸ’Ž Donation & Sponsor + +

+ Osmedeus + +

Check out for a couple of donation methods here to get a premium package

+

+ +## License + +`Osmedeus` is made with ā™„ by [@j3ssiejjj](https://twitter.com/j3ssiejjj) and it is released under the MIT license. diff --git a/cmd/cloud.go b/cmd/cloud.go new file mode 100644 index 0000000..061c876 --- /dev/null +++ b/cmd/cloud.go @@ -0,0 +1,185 @@ +package cmd + +import ( + "fmt" + "io" + "os" + "path" + + "github.com/fatih/color" + "github.com/j3ssie/osmedeus/core" + "github.com/j3ssie/osmedeus/distribute" + "github.com/j3ssie/osmedeus/libs" + "github.com/j3ssie/osmedeus/utils" + "github.com/spf13/cobra" + "golang.org/x/text/cases" + "golang.org/x/text/language" +) + +func init() { + var cloudCmd = &cobra.Command{ + Use: "cloud", + Short: "Perform a scan using the distributed cloud mode", + Long: core.Banner(), + RunE: runCloud, + } + + // core options + cloudCmd.Flags().StringVarP(&options.Cloud.Module, "module", "m", "", "module name for running") + cloudCmd.Flags().StringVarP(&options.Cloud.Flow, "flow", "f", "general", "Flow name for running (default: general)") + cloudCmd.Flags().StringVarP(&options.Cloud.Workspace, "workspace", "w", "", "Name of workspace (default is same as target)") + cloudCmd.Flags().StringSliceVarP(&options.Cloud.Params, "params", "p", []string{}, "Custom params -p='foo=bar' (Multiple -p flags are accepted)") + + // chunk inputs + cloudCmd.Flags().BoolVar(&options.Cloud.EnablePrivateIP, "privateIP", false, "Enable Private IP") + cloudCmd.Flags().BoolVar(&options.Cloud.TargetAsFile, "as-file", false, "Run target as file (use -T targets.txt file instead of -t targets.txt at cloud instance)") + cloudCmd.Flags().StringVar(&options.Cloud.LocalSyncFolder, "rfolder", "", "Remote Folder to sync back to local") + + // commands on cloud + cloudCmd.Flags().IntVar(&options.Cloud.Threads, "cloud-threads", 1, "Concurrency level on remote cloud") + cloudCmd.Flags().StringVar(&options.Cloud.Extra, "extra", "", "append raw command after the command builder") + cloudCmd.Flags().StringVar(&options.Cloud.RawCommand, "cmd", "", "specific raw command and override everything (eg: --cmd 'curl {{Target}}')") + cloudCmd.Flags().StringVar(&options.Cloud.ClearTime, "clear", "10m", "time to wait before next clear check") + cloudCmd.Flags().StringVar(&options.Cloud.TempTarget, "tempTargets", "/tmp/osm-tmp-inputs/", "Temp Folder to store targets file") + + // mics option + cloudCmd.Flags().BoolVar(&options.Cloud.EnableSyncWorkflow, "sync-workflow", false, "Enable Sync Workflow folder to remote machine first before starting the scan") + cloudCmd.Flags().BoolVarP(&options.Cloud.CopyWorkspaceToGit, "gws", "G", false, "Enable Copy Workspace to Git (run -f sync after done)") + cloudCmd.Flags().BoolVarP(&options.Cloud.DisableLocalSync, "no-lsync", "z", false, "Disable sync back data to local machine") + cloudCmd.Flags().BoolVar(&options.Cloud.BackgroundRun, "bg", false, "Send command to instance without checking if process is done or not") + cloudCmd.Flags().BoolVar(&options.Cloud.EnableTerraform, "tf", false, "Use terraform to create cloud instance") + cloudCmd.Flags().BoolVar(&options.Cloud.NoDelete, "no-del", false, "Don't delete instance after done (you can run 'osmedeus provider health' to clean up later)") + cloudCmd.Flags().BoolVar(&options.Cloud.IgnoreProcess, "no-ps", false, "Disable checking process on remote machine") + cloudCmd.Flags().IntVar(&options.Cloud.Retry, "retry", 10, "Number of retry when command is error") + cloudCmd.SetHelpFunc(CloudHelp) + RootCmd.AddCommand(cloudCmd) + cloudCmd.PreRun = func(cmd *cobra.Command, args []string) { + if options.FullHelp { + cmd.Help() + os.Exit(0) + } + } +} + +func runCloud(cmd *cobra.Command, _ []string) error { + // DBInit() + utils.GoodF("%v %v by %v", cases.Title(language.Und, cases.NoLower).String(libs.BINARY), libs.VERSION, color.HiMagentaString(libs.AUTHOR)) + utils.InforF("Storing the log file to: %v", color.CyanString(options.LogFile)) + + // parse some arguments + threads, _ := cmd.Flags().GetInt("thread") + if threads > 1 || options.Cloud.Threads <= 1 { + options.Cloud.Threads = threads + } + + // get pre run commands + getPreRun(&options) + + // change targets list if chunk mode enable + if options.Cloud.EnableChunk { + utils.InforF("Running cloud scan in chunk mode") + for _, target := range options.Scan.Inputs { + chunkTargets := HandleChunksInputs(target) + if len(chunkTargets) == 0 { + continue + } + + distribute.InitCloud(options, chunkTargets) + // remove chunk inputs + utils.DebugF("Remove chunk inputs file") + for _, ctarget := range chunkTargets { + os.RemoveAll(ctarget) + } + } + return nil + } + + // @NOTE: pro-tips + if options.Concurrency > 1 && len(options.Scan.Inputs) == 1 { + if !utils.FileExists(options.Scan.Inputs[0]) { + utils.WarnF("You're using %v in cloud scan but your input %v is just a single domain", color.HiMagentaString(`'-c %v'`, options.Concurrency), color.HiMagentaString(`'-t %v'`, options.Scan.Inputs[0])) + utils.WarnF("Consider running: osmedeus cloud -c 5 -T list-of-targets.txt") + } + } + + distribute.InitCloud(options, options.Scan.Inputs) + return nil +} + +// HandleChunksInputs split the inputs to multiple file first +func HandleChunksInputs(target string) []string { + var chunkTargets []string + utils.MakeDir(options.Cloud.ChunkInputs) + + if !utils.FileExists(target) { + utils.ErrorF("error to split input file: %v", target) + return chunkTargets + } + + if options.Cloud.NumberOfParts == 0 { + options.Cloud.NumberOfParts = options.Concurrency + } + + utils.DebugF("Splitting %v to %v part", target, options.Cloud.NumberOfParts) + rawChunks, err := utils.SplitLineChunks(target, options.Cloud.NumberOfParts) + if err != nil || len(rawChunks) == 0 { + utils.ErrorF("error to split input file: %v", target) + return chunkTargets + } + fp, err := os.Open(target) + if err != nil { + utils.ErrorF("error to open input file: %v", target) + return chunkTargets + } + for index, offset := range rawChunks { + targetName := fmt.Sprintf("%s-chunk-%v", utils.CleanPath(target), index) + targetName = path.Join(options.Cloud.ChunkInputs, targetName) + + sectionReader := io.NewSectionReader(fp, offset.Start, offset.Stop-offset.Start) + targetFile, err := os.OpenFile(targetName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + utils.ErrorF("error when create chunk file: %v", target) + continue + } + + _, err = io.Copy(targetFile, sectionReader) + if err != nil { + utils.ErrorF("error to read chunk file: %s", err) + continue + } + targetFile.Close() + chunkTargets = append(chunkTargets, targetName) + } + + return chunkTargets +} + +func getPreRun(options *libs.Options) { + if options.Cloud.Module != "" { + module := core.DirectSelectModule(*options, options.Cloud.Module) + if module == "" { + utils.ErrorF("Error to select module: %s", options.Cloud.Module) + return + } + parsedModule, err := core.ParseModules(module) + if err == nil { + options.Cloud.RemotePreRun = parsedModule.RemotePreRun + options.Cloud.LocalPostRun = parsedModule.LocalPostRun + options.Cloud.LocalPreRun = parsedModule.LocalPreRun + options.Cloud.LocalSteps = parsedModule.LocalSteps + } + return + } + + if options.Cloud.Flow != "" { + flows := core.SelectFlow(options.Cloud.Flow, *options) + for _, flow := range flows { + parseFlow, err := core.ParseFlow(flow) + if err == nil { + options.Cloud.RemotePreRun = parseFlow.RemotePreRun + options.Cloud.LocalPostRun = parseFlow.LocalPostRun + options.Cloud.LocalPreRun = parseFlow.LocalPreRun + } + } + } +} diff --git a/cmd/config.go b/cmd/config.go new file mode 100644 index 0000000..3244a84 --- /dev/null +++ b/cmd/config.go @@ -0,0 +1,100 @@ +package cmd + +import ( + "fmt" + "os" + "sort" + "strings" + + "github.com/fatih/color" + "github.com/j3ssie/osmedeus/execution" + + "github.com/j3ssie/osmedeus/core" + "github.com/j3ssie/osmedeus/utils" + "github.com/spf13/cobra" +) + +func init() { + var configCmd = &cobra.Command{ + Use: "config", + Short: "Do some configuration from CLI", + Long: core.Banner(), + RunE: runConfig, + } + + configCmd.Flags().StringP("action", "a", "", "Action") + configCmd.Flags().String("pluginsRepo", "git@gitlab.com:j3ssie/osmedeus-plugins.git", "Osmedeus Plugins repository") + // for cred action + configCmd.Flags().String("user", "", "Username") + configCmd.Flags().String("pass", "", "Password") + configCmd.Flags().StringP("workspace", "w", "", "Name of workspace") + configCmd.SetHelpFunc(ConfigHelp) + RootCmd.AddCommand(configCmd) +} + +func runConfig(cmd *cobra.Command, args []string) error { + sort.Strings(args) + action, _ := cmd.Flags().GetString("action") + workspace, _ := cmd.Flags().GetString("workspace") + + // backward compatible + if action == "" && len(args) > 0 { + action = args[0] + } + + switch action { + case "init": + if utils.FolderExists(fmt.Sprintf("%vcore", options.Env.RootFolder)) { + utils.GoodF("Look like you got properly setup.") + } + break + case "cred": + username, _ := cmd.Flags().GetString("user") + password, _ := cmd.Flags().GetString("pass") + utils.GoodF("Create new credentials %v:%v \n", username, password) + break + + case "reload": + fmt.Println("šŸ’¬ Reload the configuration will replace current settings with new ones based on the current environment") + var input string + fmt.Printf(color.HiRedString("šŸŒ€ Do you want to proceed? (y/N): ")) + fmt.Scan(&input) + input = strings.ToLower(input) + if input == "yes" || input == "y" { + utils.InforF("Delete current config and generate a new one") + os.Remove(options.ConfigFile) + os.Remove(options.TokenConfigFile) + core.InitConfig(&options) + core.ParsingConfig(&options) + } + break + + case "delete", "del": + options.Scan.Input = workspace + options.Scan.ROptions = core.ParseInput(options.Scan.Input, options) + utils.InforF("Delete Workspace: %v", options.Scan.ROptions["Workspace"]) + os.RemoveAll(options.Scan.ROptions["Output"]) + break + + case "pull": + for repo := range options.Storages { + execution.PullResult(repo, options) + } + break + + case "set": + core.SetTactic(&options) + break + case "update": + core.Update(options) + break + + case "clean", "cl", "c": + break + default: + utils.ErrorF("Unknown action: %v", color.HiRedString(action)) + fmt.Println(ConfigUsage()) + } + + return nil +} diff --git a/cmd/exec.go b/cmd/exec.go new file mode 100644 index 0000000..6b8c1d5 --- /dev/null +++ b/cmd/exec.go @@ -0,0 +1,56 @@ +package cmd + +import ( + "os" + + "github.com/j3ssie/osmedeus/core" + "github.com/j3ssie/osmedeus/utils" + "github.com/spf13/cobra" +) + +func init() { + var execCmd = &cobra.Command{ + Use: "exec", + Short: "Execute inline osmedeus scripts", + Long: core.Banner(), + RunE: runExec, + } + + execCmd.Flags().String("script", "", "Scripts to run (Multiple -s flags are accepted)") + execCmd.Flags().StringP("scriptFile", "S", "", "File contain list of scripts") + RootCmd.AddCommand(execCmd) +} + +func runExec(cmd *cobra.Command, _ []string) error { + script, _ := cmd.Flags().GetString("script") + scriptFile, _ := cmd.Flags().GetString("scriptFile") + + var scripts []string + if script != "" { + scripts = append(scripts, script) + } + if scriptFile != "" { + moreScripts := utils.ReadingFileUnique(scriptFile) + if len(moreScripts) > 0 { + scripts = append(scripts, moreScripts...) + } + } + + if len(scripts) == 0 { + utils.ErrorF("No scripts provided") + os.Exit(0) + } + runner, _ := core.InitRunner("example.com", options) + + for _, t := range options.Scan.Inputs { + // start to run scripts + options.Scan.ROptions = core.ParseInput(t, options) + for _, rscript := range scripts { + script = core.ResolveData(rscript, options.Scan.ROptions) + utils.InforF("Script: %v", script) + runner.RunScript(script) + } + } + + return nil +} diff --git a/cmd/health.go b/cmd/health.go new file mode 100644 index 0000000..5fc0c84 --- /dev/null +++ b/cmd/health.go @@ -0,0 +1,283 @@ +package cmd + +import ( + "fmt" + "os" + "path" + "sort" + + "github.com/fatih/color" + "github.com/j3ssie/osmedeus/core" + "github.com/j3ssie/osmedeus/execution" + "github.com/j3ssie/osmedeus/libs" + "github.com/j3ssie/osmedeus/utils" + "github.com/olekukonko/tablewriter" + "github.com/spf13/cobra" +) + +func init() { + var healthCmd = &cobra.Command{ + Use: "health", + Aliases: []string{"hea", "heal", "health", "healht"}, + Short: "Run diagnostics to check configurations", + Long: core.Banner(), + RunE: runHealth, + } + RootCmd.AddCommand(healthCmd) + healthCmd.PreRun = func(cmd *cobra.Command, args []string) { + if options.FullHelp { + cmd.Help() + os.Exit(0) + } + } +} + +func runHealth(_ *cobra.Command, args []string) error { + if options.PremiumPackage { + fmt.Printf("šŸ’  Osmedeus Premium %s: Run diagnostics to ensure that everything is in arrange.\n", libs.VERSION) + } else { + fmt.Printf("šŸš€ Osmedeus %s: Run diagnostics to ensure that everything is in arrange.\n", libs.VERSION) + } + + sort.Strings(args) + var err error + for _, arg := range args { + switch arg { + case "store", "git", "storages", "stora": + err = checkStorages() + case "cloud", "dist", "provider": + err = checkCloud() + case "all", "a", "full": + err = checkStorages() + if err != nil { + fmt.Println(color.YellowString("āš ļøļø There is might be something wrong with your storages: %v\n", err)) + } + err = checkCloud() + if err != nil { + fmt.Println(color.YellowString("%s If you install osmedeus on a single machine then it's okay to ignore the cloud setup\n", "[!] Cloud config setup incorrectly.")) + } + err = generalCheck() + if err != nil { + fmt.Printf("ā€¼ļø There is might be something wrong with your setup: %v\n", color.HiRedString("%v", err)) + return nil + } + break + } + if err != nil { + fmt.Println(color.YellowString("āš ļøļø There is might be something wrong with your cloud or storages setup: %v\n", err)) + return nil + } + } + if len(args) > 0 { + return nil + } + + if err = generalCheck(); err != nil { + fmt.Printf("ā€¼ļø There is might be something wrong with your setup: %v\n", err) + return nil + } + + if err = listFlows(); err != nil { + fmt.Printf("ā€¼ļø There is might be something wrong with your workflow setup: %v\n", err) + return nil + } + + if err = listDefaultModules(); err != nil { + fmt.Printf("ā€¼ļø There is might be something wrong with your workflow setup: %v\n", err) + return nil + } + fmt.Printf(color.GreenString("\nšŸ¦¾ Everything is in order. Happy Hacking šŸ¦¾\n")) + return nil +} + +func checkCloud() error { + // check packer program + if _, err := utils.RunCommandWithErr("packer -h"); err != nil { + if _, err := utils.RunCommandWithErr(fmt.Sprintf("%s -h", path.Join(options.Env.BinariesFolder, "packer"))); err != nil { + color.Red("[-] Packer program setup incorrectly") + return fmt.Errorf("error checking core programs: %v", fmt.Sprintf("%s -h", path.Join(options.Env.BinariesFolder, "packer"))) + } + } + + // check config files + if !utils.FileExists(options.CloudConfigFile) { + return fmt.Errorf("distributed cloud config doesn't exist: %v", path.Join(options.Env.CloudConfigFolder, "provider.yaml")) + } + if utils.DirLength(path.Join(options.Env.CloudConfigFolder, "providers")) == 0 { + return fmt.Errorf("providers file doesn't exist: %v", path.Join(options.Env.CloudConfigFolder, "providers")) + } + + // check SSH Keys + if !utils.FileExists(options.Cloud.SecretKey) { + keysDir := path.Dir(options.Cloud.SecretKey) + os.RemoveAll(keysDir) + utils.MakeDir(keysDir) + utils.DebugF("Generate SSH Key at: %v", options.Cloud.SecretKey) + if _, err := utils.RunCommandWithErr(fmt.Sprintf(`ssh-keygen -t ed25519 -f %s -q -N ''`, options.Cloud.SecretKey)); err != nil { + color.Red("[-] error generated SSH Key for cloud config at: %v", options.Cloud.SecretKey) + return fmt.Errorf("[-] error generated SSH Key for cloud config at: %v", options.Cloud.SecretKey) + } + } + if !utils.FileExists(options.Cloud.PublicKey) { + return fmt.Errorf("providers SSH Key missing: %v", options.Cloud.PublicKey) + } + + fmt.Printf("[+] Health Check Cloud Config: %s\n", color.GreenString("āœ”")) + return nil +} + +func checkStorages() error { + utils.DebugF("Checking storages setup") + if !execution.ValidGitURL(options.Storages["summary_repo"]) { + return fmt.Errorf("invalid git summary: %v", options.Storages["summary_repo"]) + } + + utils.DebugF("Check if your summary directory is exist or not: %v", options.Env.StoragesFolder) + if utils.DirLength(options.Env.StoragesFolder) < 1 { + return fmt.Errorf("storages folder doesn't exist: %v", options.Env.StoragesFolder) + } + + utils.DebugF("Check the secret key for git usage: %v", options.Storages["secret_key"]) + if !utils.FileExists(options.Storages["secret_key"]) { + return fmt.Errorf("secret key for git command doesn't exist: %v", options.Storages["secret_key"]) + } + + fmt.Printf("[+] Health Check Storages Config: %s\n", color.GreenString("āœ”")) + return nil +} + +func generalCheck() error { + exist := utils.FolderExists(options.Env.BaseFolder) + if !exist { + color.Red("[-] Core folder setup incorrect: %v", options.Env.BaseFolder) + return fmt.Errorf("error running diagnostics") + } + + // check core programs + var err error + // if _, err = utils.RunCommandWithErr("jaeles -h"); err != nil { + // color.Red("[-] Core program setup incorrectly") + // return fmt.Errorf("error checking core programs: %v", "jaeles") + // } + if _, err = utils.RunCommandWithErr("timeout --help"); err != nil { + color.Red("[-] Core program setup incorrectly") + return fmt.Errorf("error checking core programs: %v", "timeout") + } + if _, err = utils.RunCommandWithErr("amass -h"); err != nil { + color.Red("[-] Core program setup incorrectly") + return fmt.Errorf("error checking core programs: %v", "amass") + } + _, err = utils.RunCommandWithErr(fmt.Sprintf("%s -h", path.Join(options.Env.BinariesFolder, "httprobe"))) + if err != nil { + color.Red("[-] Core program setup incorrectly") + return fmt.Errorf("error checking core programs: %v", fmt.Sprintf("%s -h", path.Join(options.Env.BinariesFolder, "httprobe"))) + } + fmt.Printf("[+] Health Check Core Programs: %s\n", color.GreenString("āœ”")) + + // Check core signatures + okVuln := false + // if utils.DirLength("~/.jaeles/base-signatures/") > 0 || utils.DirLength("~/pro-signatures/") > 0 { + // okVuln = true + // } + + if utils.DirLength("~/nuclei-templates") > 0 { + okVuln = true + } + + if okVuln { + fmt.Printf("[+] Health Check Vulnerability scanning config: %s\n", color.GreenString("āœ”")) + } else { + color.Red("vulnerability scanning config setup incorrectly") + return fmt.Errorf("vulnerability scanning config setup incorrectly") + } + + // check data folder + if utils.FolderExists(options.Env.DataFolder) { + fmt.Printf("[+] Health Check Data Config: %s\n", color.GreenString("āœ”")) + } else { + color.Red("[-] Data setup incorrectly: %v", options.Env.DataFolder) + return fmt.Errorf("[-] Data setup incorrectly: %v", options.Env.DataFolder) + } + return nil +} + +func listFlows() error { + flows := core.ListFlow(options) + if len(flows) == 0 { + color.Red("[-] Error to list workflows: %s", options.Env.WorkFlowsFolder) + return fmt.Errorf("[-] Error to list workflows: %s", options.Env.WorkFlowsFolder) + } + fmt.Printf("[+] Health Check Workflows: %s\n", color.GreenString("āœ”")) + if options.PremiumPackage { + fmt.Printf("šŸ’Ž Making use of the premium workflow\n") + } + + var content [][]string + for _, flow := range flows { + parsedFlow, err := core.ParseFlow(flow) + if err != nil { + utils.ErrorF("Error parsing flow: %v", flow) + continue + } + + if parsedFlow.SkipIndexed { + continue + } + + row := []string{ + parsedFlow.Name, parsedFlow.Desc, + } + + content = append(content, row) + } + fmt.Printf("\nFound %v available workflows at: %s \n\n", color.HiGreenString("%v", len(content)), color.HiCyanString(options.Env.WorkFlowsFolder)) + + table := tablewriter.NewWriter(os.Stdout) + table.SetAutoFormatHeaders(false) + table.SetHeader([]string{"Flow Name", "Description"}) + table.SetBorders(tablewriter.Border{Left: true, Top: true, Right: true, Bottom: true}) + table.SetColWidth(120) + table.AppendBulk(content) // Add Bulk Data + table.Render() + + h := color.HiCyanString("\nUsage:\n") + h += color.HiGreenString(" osmedeus scan -f %v", color.HiMagentaString("[flowName]")) + color.HiGreenString(" -t ") + color.HiMagentaString("[target]") + "\n" + fmt.Printf(h) + return nil +} + +func listDefaultModules() error { + defaultModule := path.Join(options.Env.WorkFlowsFolder, "default-modules") + modules := core.DefaultWorkflows(options) + + if len(modules) == 0 { + return fmt.Errorf("[-] Error to list default modules: %s", defaultModule) + } + + var content [][]string + for _, flow := range modules { + parsedModule, err := core.ParseModules(flow) + if err != nil { + utils.ErrorF("Error parsing flow: %v", flow) + continue + } + row := []string{ + parsedModule.Name, parsedModule.Desc, + } + content = append(content, row) + } + fmt.Printf("\nFound %v default modules at: %s \n\n", color.HiGreenString("%v", len(content)), color.HiCyanString(defaultModule)) + + table := tablewriter.NewWriter(os.Stdout) + table.SetAutoFormatHeaders(false) + table.SetHeader([]string{"Module Name", "Description"}) + table.SetBorders(tablewriter.Border{Left: true, Top: true, Right: true, Bottom: true}) + table.SetColWidth(120) + table.AppendBulk(content) // Add Bulk Data + table.Render() + + h := color.HiCyanString("\nModule Usage:\n") + h += color.HiGreenString(" osmedeus scan -m %v", color.HiMagentaString("[moduleName]")) + color.HiGreenString(" -t ") + color.HiMagentaString("[target]") + "\n\n" + fmt.Printf(h) + return nil +} diff --git a/cmd/provider.go b/cmd/provider.go new file mode 100644 index 0000000..6a2c029 --- /dev/null +++ b/cmd/provider.go @@ -0,0 +1,266 @@ +package cmd + +import ( + "fmt" + "os" + "sync" + "time" + + "github.com/j3ssie/osmedeus/core" + "github.com/j3ssie/osmedeus/distribute" + "github.com/j3ssie/osmedeus/provider" + "github.com/j3ssie/osmedeus/utils" + "github.com/olekukonko/tablewriter" + "github.com/panjf2000/ants" + "github.com/spf13/cobra" +) + +func init() { + var providerCmd = &cobra.Command{ + Use: "provider", + Aliases: []string{"provide", "pro"}, + Short: "Cloud utils for Distributed Mode", + Long: core.Banner(), + RunE: runProvider, + } + + providerCmd.PersistentFlags().StringVar(&options.Cloud.RawCommand, "cmd", "", "raw command") + providerCmd.PersistentFlags().BoolVar(&options.Cloud.CheckingLimit, "check", false, "Only check for limit of config") + providerCmd.PersistentFlags().StringVar(&options.Cloud.InstanceName, "name", "", "override instance name") + providerCmd.PersistentFlags().BoolVar(&options.Cloud.BackgroundRun, "bg", false, "Send command to instance and run it in background") + providerCmd.PersistentFlags().BoolVar(&options.Cloud.IgnoreConfigFile, "ic", false, "Ignore token in the config file") + providerCmd.PersistentFlags().IntVar(&options.Cloud.Retry, "retry", 10, "Number of retry when command is error") + providerCmd.PersistentFlags().StringSlice("id", []string{}, "Instance IDs that will be delete") + providerCmd.Flags().StringVar(&options.Cloud.ClearTime, "clear", "10m", "time to wait before next clear check") + providerCmd.PersistentFlags().BoolVar(&options.Cloud.ForEverHealthCheck, "for", false, "Continuesly running the health check forever") + + var providerWizard = &cobra.Command{ + Use: "wizard", + Aliases: []string{"wi", "wiz", "wizazrd"}, + Short: "Start a cloud config wizard", + Long: core.Banner(), + RunE: runCloudInit, + } + providerWizard.PersistentFlags().BoolVar(&options.Cloud.AddNewProvider, "add", false, "Open wizard to add new provider only") + + var providerBuild = &cobra.Command{ + Use: "build", + Aliases: []string{"buil"}, + Short: "Build snapshot image", + Long: core.Banner(), + RunE: runProviderBuild, + } + var providerCreate = &cobra.Command{ + Use: "create", + Aliases: []string{"cre"}, + Short: "Create cloud instance based on image", + Long: core.Banner(), + RunE: runProviderCreate, + } + + var providerHealth = &cobra.Command{ + Use: "health", + Aliases: []string{"hea", "heal", "health", "healht"}, + Short: "Conduct a health assessment on cloud instances that are currently operational", + Long: core.Banner(), + RunE: runCloudHealth, + } + + var providerValidate = &cobra.Command{ + Use: "validate", + Aliases: []string{"val"}, + Short: "Run validate of the existing cloud configs", + Long: core.Banner(), + RunE: runProviderValidate, + } + var providerList = &cobra.Command{ + Use: "list", + Aliases: []string{"ls"}, + Short: "List all running instances", + Long: core.Banner(), + RunE: runProviderListing, + } + var providerDel = &cobra.Command{ + Use: "delete", + Aliases: []string{"del"}, + Short: "Delete instances by id", + Long: core.Banner(), + RunE: runProviderDelete, + } + var providerClear = &cobra.Command{ + Use: "clear", + Aliases: []string{"clea", "clr"}, + Short: "Clear all instances in the instances folders", + Long: core.Banner(), + RunE: runProviderClear, + } + providerCmd.AddCommand(providerWizard) + providerCmd.AddCommand(providerList) + providerCmd.AddCommand(providerDel) + providerCmd.AddCommand(providerValidate) + providerCmd.AddCommand(providerHealth) + providerCmd.AddCommand(providerCreate) + providerCmd.AddCommand(providerBuild) + providerCmd.AddCommand(providerClear) + providerCmd.SetHelpFunc(CloudHelp) + RootCmd.AddCommand(providerCmd) + providerCmd.PreRun = func(cmd *cobra.Command, args []string) { + if options.FullHelp { + cmd.Help() + os.Exit(0) + } + } +} + +func runCloudHealth(_ *cobra.Command, _ []string) error { + distribute.CheckingCloudInstance(options) + if options.Cloud.ForEverHealthCheck { + for { + distribute.CheckingCloudInstance(options) + waitTime := utils.CalcTimeout(options.Cloud.ClearTime) + time.Sleep(time.Duration(waitTime) * time.Second) + } + } + + return nil +} + +func runProviderClear(_ *cobra.Command, _ []string) error { + distribute.ClearAllInstances(options) + return nil +} + +func runCloudInit(_ *cobra.Command, _ []string) error { + // interactive mode to show config file here + distribute.InitCloudSetup(options) + return nil +} + +func runProvider(_ *cobra.Command, args []string) error { + if len(args) == 0 { + fmt.Println(CloudUsage()) + } + return nil +} + +func runProviderBuild(_ *cobra.Command, _ []string) error { + options.Cloud.OnlyCreateDroplet = true + options.Cloud.ReBuildBaseImage = true + + // building multiple tokens + options.Cloud.TokensFile = utils.NormalizePath(options.Cloud.TokensFile) + if options.Cloud.TokensFile != "" { + tokens := utils.ReadingFileUnique(options.Cloud.TokensFile) + if len(tokens) == 0 { + utils.ErrorF("token file not found: %v", options.Cloud.TokensFile) + return nil + } + + var wg sync.WaitGroup + p, _ := ants.NewPoolWithFunc(options.Concurrency, func(i interface{}) { + lOptions := options + lOptions.Cloud.Token = i.(string) + + distribute.InitCloud(lOptions, lOptions.Scan.Inputs) + wg.Done() + }, ants.WithPreAlloc(true)) + defer p.Release() + + for _, token := range tokens { + wg.Add(1) + _ = p.Invoke(token) + } + wg.Wait() + return nil + + } + + distribute.InitCloud(options, options.Scan.Inputs) + return nil +} + +func runProviderCreate(_ *cobra.Command, _ []string) error { + options.Cloud.OnlyCreateDroplet = true + if len(options.Scan.Inputs) == 0 { + options.Scan.Inputs = append(options.Scan.Inputs, utils.RandomString(4)) + } + + distribute.InitCloud(options, options.Scan.Inputs) + return nil +} + +func runProviderValidate(_ *cobra.Command, _ []string) error { + cloudValidate() + return nil +} + +func runProviderListing(_ *cobra.Command, _ []string) error { + options.Cloud.BackgroundRun = true + utils.InforF("Listing all instances in the cloud provider(s) ...") + cloudRunners := distribute.GetClouds(options) + cloudListing(cloudRunners) + return nil +} + +func runProviderDelete(cmd *cobra.Command, _ []string) error { + options.Cloud.OnlyCreateDroplet = true + options.Cloud.BackgroundRun = true + cloudRunners := distribute.GetClouds(options) + InstanceIDs, _ := cmd.Flags().GetStringSlice("id") + + for _, InstanceID := range InstanceIDs { + for _, cloudRunner := range cloudRunners { + cloudRunner.Provider.DeleteInstance(InstanceID) + } + } + + cloudListing(cloudRunners) + return nil +} + +func cloudListing(cloudRunners []distribute.CloudRunner) { + var content [][]string + for _, cloudRunner := range cloudRunners { + cloudRunner.Provider.Action(provider.ListInstance) + for _, instance := range cloudRunner.Provider.Instances { + row := []string{ + cloudRunner.Provider.ProviderName, + cloudRunner.Provider.RedactedToken, + instance.InstanceID, + instance.InstanceName, + instance.IPAddress, + } + content = append(content, row) + } + } + table := tablewriter.NewWriter(os.Stderr) + table.SetAutoFormatHeaders(false) + table.SetHeader([]string{"Provider", "Token", "Instance ID", "Instance Name", "IP Address"}) + table.SetBorders(tablewriter.Border{Left: true, Top: false, Right: true, Bottom: false}) + table.SetCenterSeparator("|") + table.AppendBulk(content) // Add Bulk Data + table.Render() +} + +func cloudValidate() { + cloudRunners := distribute.GetClouds(options) + + var content [][]string + for _, cloudRunner := range cloudRunners { + row := []string{ + cloudRunner.Provider.ProviderName, + cloudRunner.Provider.RedactedToken, + cloudRunner.Provider.SSHKeyID, + cloudRunner.Provider.SnapshotID, + } + content = append(content, row) + } + table := tablewriter.NewWriter(os.Stderr) + table.SetAutoFormatHeaders(false) + table.SetHeader([]string{"Provider", "Token", "SSH Key ID", "Osmedeus Snapshot ID"}) + table.SetBorders(tablewriter.Border{Left: true, Top: false, Right: true, Bottom: false}) + table.SetCenterSeparator("|") + table.AppendBulk(content) // Add Bulk Data + table.Render() + +} diff --git a/cmd/queue.go b/cmd/queue.go new file mode 100644 index 0000000..bb79070 --- /dev/null +++ b/cmd/queue.go @@ -0,0 +1,87 @@ +package cmd + +import ( + "fmt" + "os" + "path" + "strings" + + "github.com/fatih/color" + "github.com/j3ssie/osmedeus/core" + "github.com/j3ssie/osmedeus/libs" + "github.com/j3ssie/osmedeus/utils" + jsoniter "github.com/json-iterator/go" + "github.com/spf13/cobra" +) + +func init() { + var queueCmd = &cobra.Command{ + Use: "queue", + Short: "Running the scan with input from queue file", + Aliases: []string{"queq", "quee", "queu", "que"}, + Long: core.Banner(), + RunE: runQueue, + } + queueCmd.PersistentFlags().StringVarP(&options.Queue.QueueFile, "queue-file", "Q", fmt.Sprintf("~/.%s/queue/queue-mimic.txt", libs.BINARY), "File contain list of target to simulate the queue") + queueCmd.PersistentFlags().BoolVar(&options.Queue.Add, "add", false, "Add new input to the queue file") + queueCmd.PersistentFlags().BoolVarP(&options.Queue.InputAsFile, "as-file", "F", false, "treat input as a file") + queueCmd.PersistentFlags().StringVar(&options.Queue.RawCommand, "cmd", "", "Raw Command to run") + queueCmd.SetHelpFunc(QueueHelp) + RootCmd.AddCommand(queueCmd) + queueCmd.PreRun = func(cmd *cobra.Command, args []string) { + if options.FullHelp { + cmd.Help() + os.Exit(0) + } + } +} + +func runQueue(_ *cobra.Command, _ []string) error { + options.Queue.QueueFile = utils.NormalizePath(options.Queue.QueueFile) + options.Queue.QueueFolder = path.Dir(options.Queue.QueueFile) + if !utils.FolderExists(options.Queue.QueueFolder) { + utils.MakeDir(options.Queue.QueueFolder) + } + + if options.Queue.Add { + addInput() + return nil + } + + if !utils.FileExists(options.Queue.QueueFile) { + utils.WriteToFile(options.Queue.QueueFile, "") + } + + content := utils.ReadingFileUnique(options.Queue.QueueFile) + if len(content) == 0 { + utils.WarnF("Queue file is empty: %v", options.Queue.QueueFile) + utils.WarnF("Consider to add a input to it:" + color.HiGreenString(" osmedeus queue --add -t example.com")) + } else { + utils.InforF("Queue file is not empty: %v", color.HiCyanString(options.Queue.QueueFile)) + utils.InforF("Consider to delete it if you want a fresh scan") + } + + core.QueueWatcher(options) + return nil +} + +func addInput() { + utils.InforF("Adding new input to the queue file: %v", color.HiCyanString(options.Queue.QueueFile)) + // osmedeus queue --add -t example.com + if options.Queue.RawCommand == "" { + utils.WriteToFile(options.Queue.QueueFile, strings.Join(options.Scan.Inputs, "\n")) + return + } + + // osmedeus queue --add -t /tmp/cidr --cmd "osmedeus -t {{Input}} -m recon -w" + for _, target := range options.Scan.Inputs { + queueInput := libs.InputFormat{ + Input: target, + Command: options.Queue.RawCommand, + InputAsFile: options.Queue.InputAsFile, + } + if line, ok := jsoniter.MarshalToString(queueInput); ok == nil { + utils.AppendToContent(options.Queue.QueueFile, line) + } + } +} diff --git a/cmd/report.go b/cmd/report.go new file mode 100644 index 0000000..b132efe --- /dev/null +++ b/cmd/report.go @@ -0,0 +1,162 @@ +package cmd + +import ( + "io" + "net/http" + "os" + "path" + "path/filepath" + "strings" + + "github.com/fatih/color" + "github.com/j3ssie/osmedeus/core" + "github.com/j3ssie/osmedeus/utils" + "github.com/spf13/cobra" +) + +func init() { + var reportCmd = &cobra.Command{ + Use: "report", + Short: "Show report of existing workspace", + Long: core.Banner(), + RunE: runReport, + } + + var lsCmd = &cobra.Command{ + Use: "list", + Aliases: []string{"ls"}, + Short: "List all current existing workspace", + Long: core.Banner(), + RunE: runReportList, + } + reportCmd.AddCommand(lsCmd) + + var viewCmd = &cobra.Command{ + Use: "view", + Aliases: []string{"vi", "v"}, + Short: "View all reports of existing workspace", + Long: core.Banner(), + RunE: runReportView, + } + reportCmd.AddCommand(viewCmd) + + var extractCmd = &cobra.Command{ + Use: "extract", + Aliases: []string{"ext", "ex", "e"}, + Short: "Extract a compressed workspace", + Long: core.Banner(), + RunE: runReportExtract, + } + extractCmd.Flags().StringVar(&options.Report.ExtractFolder, "dest", "", "Destination folder to extract data to") + reportCmd.AddCommand(extractCmd) + + var compressCmd = &cobra.Command{ + Use: "compress", + Aliases: []string{"com", "compr", "compres", "c"}, + Short: "Create a backup of the selected workspace", + Long: core.Banner(), + RunE: runReportCompress, + } + reportCmd.AddCommand(compressCmd) + + reportCmd.PersistentFlags().BoolVar(&options.Report.Raw, "raw", false, "Show all the file in the workspace") + reportCmd.PersistentFlags().StringVar(&options.Report.PublicIP, "ip", "", "Show downloadable file with the given IP address") + reportCmd.PersistentFlags().BoolVar(&options.Report.Static, "static", false, "Show report file with Prefix Static") + reportCmd.SetHelpFunc(ReportHelp) + RootCmd.AddCommand(reportCmd) + reportCmd.PreRun = func(cmd *cobra.Command, args []string) { + if options.FullHelp { + cmd.Help() + os.Exit(0) + } + } +} + +func runReportList(_ *cobra.Command, _ []string) error { + core.ListWorkspaces(options) + return nil +} + +func runReportView(_ *cobra.Command, _ []string) error { + if options.Report.PublicIP == "" { + if utils.GetOSEnv("IPAddress", "127.0.0.1") == "127.0.0.1" { + options.Report.PublicIP = utils.GetOSEnv("IPAddress", "127.0.0.1") + } + } + + if options.Report.PublicIP == "0" || options.Report.PublicIP == "0.0.0.0" { + options.Report.PublicIP = getPublicIP() + } + + if len(options.Scan.Inputs) == 0 { + core.ListWorkspaces(options) + utils.InforF("Please select workspace to view report. Try %s", color.HiCyanString(`'osmedeus report view -t target.com'`)) + return nil + } + + for _, target := range options.Scan.Inputs { + core.ListSingleWorkspace(options, target) + } + return nil +} + +func runReportExtract(_ *cobra.Command, _ []string) error { + var err error + if options.Report.ExtractFolder == "" { + options.Report.ExtractFolder = options.Env.WorkspacesFolder + } else { + options.Report.ExtractFolder, err = filepath.Abs(filepath.Dir(options.Report.ExtractFolder)) + if err != nil { + return err + } + } + + for _, input := range options.Scan.Inputs { + core.ExtractBackup(input, options) + + target := strings.ReplaceAll(path.Base(input), ".tar.gz", "") + core.ListSingleWorkspace(options, target) + } + + return nil +} + +func runReportCompress(_ *cobra.Command, _ []string) error { + for _, target := range options.Scan.Inputs { + core.CompressWorkspace(target, options) + } + return nil +} + +func runReport(_ *cobra.Command, args []string) error { + if options.Report.PublicIP == "" { + if utils.GetOSEnv("IPAddress", "127.0.0.1") == "127.0.0.1" { + options.Report.PublicIP = utils.GetOSEnv("IPAddress", "127.0.0.1") + } + } + + if options.Report.PublicIP == "0" || options.Report.PublicIP == "0.0.0.0" { + options.Report.PublicIP = getPublicIP() + } + + if len(args) == 0 { + core.ListWorkspaces(options) + } + + return nil +} + +func getPublicIP() string { + utils.DebugF("getting Public IP Address") + req, err := http.Get("https://api.ipify.org") + if err != nil { + return "127.0.0.1" + } + defer req.Body.Close() + + body, err := io.ReadAll(req.Body) + if err != nil { + return "127.0.0.1" + } + return string(body) +} diff --git a/cmd/root.go b/cmd/root.go new file mode 100644 index 0000000..57e07ec --- /dev/null +++ b/cmd/root.go @@ -0,0 +1,147 @@ +package cmd + +import ( + "bufio" + "fmt" + "os" + "strings" + + "github.com/j3ssie/osmedeus/core" + "github.com/j3ssie/osmedeus/libs" + "github.com/j3ssie/osmedeus/utils" + "github.com/spf13/cobra" +) + +var options = libs.Options{} +var RootCmd = &cobra.Command{ + Use: libs.BINARY, + Short: fmt.Sprintf("%s - %s", libs.BINARY, libs.DESC), + Long: core.Banner(), +} + +// Execute main function +func Execute() { + if err := RootCmd.Execute(); err != nil { + fmt.Println(err) + os.Exit(1) + } +} + +func init() { + RootCmd.PersistentFlags().StringVar(&options.Env.RootFolder, "rootFolder", fmt.Sprintf("~/.%s/", libs.BINARY), "The main folder where all configurations are stored") + RootCmd.PersistentFlags().StringVar(&options.Env.BaseFolder, "baseFolder", fmt.Sprintf("~/%s-base/", libs.BINARY), "Base Folder which is store data, binaries and workflows") + RootCmd.PersistentFlags().StringVar(&options.Env.DataFolder, "dataFolder", fmt.Sprintf("~/%s-base/data", libs.BINARY), "Data folder which is store wordlists, payloads, etc") + RootCmd.PersistentFlags().StringVar(&options.Env.WorkFlowsFolder, "wfFolder", "", fmt.Sprintf("Custom Workflow folder (default will get from '$HOME/%s-base/workflow')", libs.BINARY)) + RootCmd.PersistentFlags().StringVar(&options.ConfigFile, "configFile", fmt.Sprintf("~/.%s/config.yaml", libs.BINARY), "Main configurations file") + + // Workspace folder + RootCmd.PersistentFlags().StringVarP(&options.Env.WorkspacesFolder, "wsFolder", "W", fmt.Sprintf("~/workspaces-%s", libs.BINARY), "The main data folder within the workspaces where all scan results are stored") + + // parse target as global flag + RootCmd.PersistentFlags().StringSliceVarP(&options.Scan.Inputs, "target", "t", []string{}, "The target you want to run/execute") + RootCmd.PersistentFlags().StringVarP(&options.Scan.InputList, "targets", "T", "", "List of target as a file") + + // Scan command + RootCmd.PersistentFlags().IntVarP(&options.Concurrency, "concurrency", "c", 1, "Concurrency level (recommend to keep it as 1 on machine has RAM smaller than 2GB)") + RootCmd.PersistentFlags().StringSliceVarP(&options.Scan.Modules, "module", "m", []string{}, "Target to running") + RootCmd.PersistentFlags().StringVarP(&options.Scan.Flow, "flow", "f", "general", "Flow name for running (default: general)") + RootCmd.PersistentFlags().StringVarP(&options.Scan.CustomWorkspace, "workspace", "w", "", "Name of workspace (default is same as target)") + RootCmd.PersistentFlags().StringSliceVarP(&options.Scan.Params, "params", "p", []string{}, "Custom params -p='foo=bar' (Multiple -p flags are accepted)") + RootCmd.PersistentFlags().StringVar(&options.Scan.SuffixName, "suffix", "", "Suffix string for file converted (default: randomly)") + RootCmd.PersistentFlags().IntVarP(&options.Threads, "threads-hold", "B", 0, "Threads hold for each module (default: number of CPUs)") + RootCmd.PersistentFlags().StringVar(&options.Tactics, "tactic", "default", "Choosing the tactic for running workflow from [default aggressive gently]") + RootCmd.PersistentFlags().StringVar(&options.Scan.ParamsFile, "params-file", "", "Custom file params --params-file=params.yaml") + + // cloud flags + RootCmd.PersistentFlags().BoolVar(&options.Cloud.EnableChunk, "chunk", false, "Enable chunk mode") + RootCmd.PersistentFlags().BoolVar(&options.Scan.RemoteCall, "from-remote", false, "Invocation from a remote machine") + RootCmd.PersistentFlags().IntVarP(&options.Cloud.NumberOfParts, "chunk-parts", "P", 0, "Number of chunks file to split (default: equal with concurrency)") + RootCmd.PersistentFlags().StringVar(&options.Cloud.ChunkInputs, "chunkFolder", "/tmp/chunk-inputs/", "Temp Folder to store chunk inputs") + RootCmd.PersistentFlags().StringVar(&options.Timeout, "timeout", "", "Global timeout for each step (e.g: 60s, 30m, 2h)") + RootCmd.PersistentFlags().StringVar(&options.Cloud.Size, "size", "", "Override Size of cloud provider (default will get from 'cloud/provider.yaml')") + RootCmd.PersistentFlags().StringVar(&options.Cloud.Region, "region", "", "Override Region of cloud provider (default will get from 'cloud/provider.yaml')") + RootCmd.PersistentFlags().StringVar(&options.Cloud.Token, "token", "", "Override token of cloud provider (default will get from 'cloud/provider.yaml')") + RootCmd.PersistentFlags().StringVar(&options.Cloud.TokensFile, "token-file", "", "File contains list token of cloud providers") + RootCmd.PersistentFlags().StringVar(&options.Cloud.Provider, "provider", "", "Provider config file (default will get from 'cloud/provider.yaml')") + RootCmd.PersistentFlags().BoolVar(&options.Cloud.ReBuildBaseImage, "rebuild", false, "Forced to rebuild the images event though the version didn't change") + + // mics option + RootCmd.PersistentFlags().StringVar(&options.LogFile, "log", "", fmt.Sprintf("Log File (default will store in '%s')", libs.LDIR)) + RootCmd.PersistentFlags().StringVarP(&options.ScanID, "sid", "s", "", "Scan ID to continue the scan without create new scan record") + RootCmd.PersistentFlags().BoolVarP(&options.Resume, "resume", "R", false, "Enable Resume mode to skip modules that have already been finished") + RootCmd.PersistentFlags().BoolVar(&options.Debug, "debug", false, "Enable Debug output") + RootCmd.PersistentFlags().BoolVarP(&options.Quite, "quite", "q", false, "Show only essential information") + RootCmd.PersistentFlags().BoolVar(&options.FullHelp, "hh", false, "Show full help message") + RootCmd.PersistentFlags().BoolVar(&options.WildCardCheck, "ww", false, "Check for wildcard target") + RootCmd.PersistentFlags().BoolVar(&options.DisableValidateInput, "nv", false, "Disable Validate Input") + RootCmd.PersistentFlags().BoolVar(&options.Update.NoUpdate, "nu", false, "Disable Update options") + RootCmd.PersistentFlags().BoolVarP(&options.EnableFormatInput, "format-input", "J", false, "Enable special input format") + RootCmd.PersistentFlags().IntVar(&options.MDCodeBlockLimit, "block-size", 10000, "Size limit for code block tags to before it's automatic truncation") + + // disable options + RootCmd.PersistentFlags().BoolVar(&options.NoNoti, "nn", false, "No notification") + RootCmd.PersistentFlags().BoolVar(&options.NoBanner, "nb", false, "No banner") + RootCmd.PersistentFlags().BoolVarP(&options.NoGit, "no-git", "N", false, "No git storage") + RootCmd.PersistentFlags().BoolVarP(&options.NoClean, "no-clean", "C", false, "No clean junk output") + RootCmd.PersistentFlags().BoolVar(&options.NoPreRun, "no-prerun", false, "Disable pre run scripts") + RootCmd.PersistentFlags().BoolVar(&options.NoPostRun, "no-postrun", false, "Disable post run scripts") + RootCmd.PersistentFlags().StringSliceVarP(&options.Exclude, "exclude", "x", []string{}, "Exclude module name (Multiple -x flags are accepted)") + RootCmd.PersistentFlags().BoolVarP(&options.CustomGit, "git", "g", false, "Use custom Git repo") + + // sync options + RootCmd.PersistentFlags().BoolVar(&options.EnableDeStorage, "des", false, "Enable Dedicated Storages") + RootCmd.PersistentFlags().BoolVar(&options.GitSync, "sync", false, "Enable Sync Check before doing git push") + RootCmd.PersistentFlags().IntVar(&options.SyncTimes, "sync-timee", 15, "Number of times to check before force push") + RootCmd.PersistentFlags().IntVar(&options.PollingTime, "poll-timee", 100, "Number of seconds to sleep before do next sync check") + RootCmd.PersistentFlags().BoolVar(&options.NoCdn, "no-cdn", false, "Disable CDN feature") + RootCmd.PersistentFlags().BoolVarP(&options.EnableBackup, "backup", "b", false, "Backup the result after the scan is done") + + // update options + RootCmd.PersistentFlags().BoolVar(&options.Update.IsUpdateBin, "bin", false, "Update binaries too") + RootCmd.PersistentFlags().BoolVar(&options.Update.EnableUpdate, "update", false, "Enable auto update") + RootCmd.PersistentFlags().StringVar(&options.Update.UpdateFolder, "update-folder", "/tmp/osm-update", "Folder to clone the update folder") + + RootCmd.SetHelpFunc(RootHelp) + cobra.OnInitialize(initConfig) + RootCmd.PreRun = func(cmd *cobra.Command, args []string) { + if options.FullHelp { + cmd.Help() + os.Exit(0) + } + } +} + +// initConfig reads in config file and ENV variables if set. +func initConfig() { + if options.JsonOutput { + options.Quite = true + } + + /* Really Start the program */ + utils.InitLog(&options) + utils.InitHTTPClient() + if err := core.InitConfig(&options); err != nil { + utils.ErrorF("config file does not writable: %v", options.ConfigFile) + utils.BlockF("fatal", "Make sure you are login as 'root user' if your installation done via root user") + } + core.ParsingConfig(&options) + + // parse inputs + if options.Scan.InputList != "" { + if utils.FileExists(options.Scan.InputList) { + options.Scan.Inputs = append(options.Scan.Inputs, utils.ReadingFileUnique(options.Scan.InputList)...) + } + } + + // detect if anything came from stdin + stat, _ := os.Stdin.Stat() + if (stat.Mode() & os.ModeCharDevice) == 0 { + sc := bufio.NewScanner(os.Stdin) + for sc.Scan() { + target := strings.TrimSpace(sc.Text()) + if err := sc.Err(); err == nil && target != "" { + options.Scan.Inputs = append(options.Scan.Inputs, target) + } + } + } +} diff --git a/cmd/scan.go b/cmd/scan.go new file mode 100644 index 0000000..24156ee --- /dev/null +++ b/cmd/scan.go @@ -0,0 +1,80 @@ +package cmd + +import ( + "os" + "strings" + "sync" + + "github.com/fatih/color" + "github.com/j3ssie/osmedeus/core" + "github.com/j3ssie/osmedeus/libs" + "github.com/j3ssie/osmedeus/utils" + "github.com/panjf2000/ants" + "github.com/spf13/cobra" + "golang.org/x/text/cases" + "golang.org/x/text/language" +) + +func init() { + var scanCmd = &cobra.Command{ + Use: "scan", + Short: "Conduct a scan following a predetermined flow/module", + Long: core.Banner(), + RunE: runScan, + } + + scanCmd.SetHelpFunc(ScanHelp) + RootCmd.AddCommand(scanCmd) + scanCmd.PreRun = func(cmd *cobra.Command, args []string) { + if options.FullHelp { + cmd.Help() + os.Exit(0) + } + } +} + +func runScan(_ *cobra.Command, _ []string) error { + utils.GoodF("%v %v by %v", cases.Title(language.Und, cases.NoLower).String(libs.BINARY), libs.VERSION, color.HiMagentaString(libs.AUTHOR)) + utils.InforF("Storing the log file to: %v", color.CyanString(options.LogFile)) + + var wg sync.WaitGroup + p, _ := ants.NewPoolWithFunc(options.Concurrency, func(i interface{}) { + // really start to scan + CreateRunner(i) + wg.Done() + }, ants.WithPreAlloc(true)) + defer p.Release() + + if options.Cloud.EnableChunk { + for _, target := range options.Scan.Inputs { + chunkTargets := HandleChunksInputs(target) + for _, chunkTarget := range chunkTargets { + wg.Add(1) + _ = p.Invoke(chunkTarget) + } + } + } else { + for _, target := range options.Scan.Inputs { + wg.Add(1) + _ = p.Invoke(strings.TrimSpace(target)) + } + } + + wg.Wait() + return nil +} + +func CreateRunner(j interface{}) { + target := j.(string) + if core.IsRootDomain(target) && options.Scan.Flow == "general" && len(options.Scan.Modules) == 0 { + utils.WarnF("looks like you scanning a subdomain '%s' with general flow. The result might be much less than usual", color.HiCyanString(target)) + utils.WarnF("Better input should be root domain with TLD like '-t target.com'") + } + + runner, err := core.InitRunner(target, options) + if err != nil { + utils.ErrorF("Error init runner with: %s", target) + return + } + runner.Start() +} diff --git a/cmd/server.go b/cmd/server.go new file mode 100644 index 0000000..1c4ebb6 --- /dev/null +++ b/cmd/server.go @@ -0,0 +1,49 @@ +package cmd + +import ( + "fmt" + "os" + + "github.com/fatih/color" + "github.com/j3ssie/osmedeus/core" + "github.com/j3ssie/osmedeus/libs" + "github.com/j3ssie/osmedeus/server" + "github.com/j3ssie/osmedeus/utils" + "github.com/spf13/cobra" + "golang.org/x/text/cases" + "golang.org/x/text/language" +) + +func init() { + var serverCmd = &cobra.Command{ + Use: "server", + Short: "Start Web Server", + Long: core.Banner(), + RunE: runServer, + } + serverCmd.Flags().String("host", "0.0.0.0", "IP address to bind the server") + serverCmd.Flags().String("port", "8000", "Port") + serverCmd.Flags().IntVar(&options.Server.PollingTime, "poll-time", 60, "Polling time to check next task") + serverCmd.Flags().BoolVar(&options.Server.DisableSSL, "disable-ssl", false, "Disable workspaces directory listing") + serverCmd.Flags().BoolVar(&options.Server.DisableWorkspaceListing, "disable-listing", false, "Disable workspaces directtory listing") + serverCmd.Flags().BoolVar(&options.Server.PreFork, "prefork", false, "Enable Prefork mode for the api server") + serverCmd.Flags().BoolVarP(&options.Server.NoAuthen, "no-auth", "A", false, "Disable authentication for the api server") + + serverCmd.SetHelpFunc(ServerHelp) + RootCmd.AddCommand(serverCmd) + serverCmd.PreRun = func(cmd *cobra.Command, args []string) { + if options.FullHelp { + cmd.Help() + os.Exit(0) + } + } +} + +func runServer(cmd *cobra.Command, _ []string) error { + host, _ := cmd.Flags().GetString("host") + port, _ := cmd.Flags().GetString("port") + options.Server.Bind = fmt.Sprintf("%v:%v", host, port) + utils.GoodF("%v %v by %v", cases.Title(language.Und, cases.NoLower).String(libs.BINARY), libs.VERSION, color.HiMagentaString(libs.AUTHOR)) + server.StartServer(options) + return nil +} diff --git a/cmd/update.go b/cmd/update.go new file mode 100644 index 0000000..a32daae --- /dev/null +++ b/cmd/update.go @@ -0,0 +1,77 @@ +package cmd + +import ( + "os" + + "github.com/j3ssie/osmedeus/core" + "github.com/j3ssie/osmedeus/utils" + "github.com/spf13/cobra" +) + +func init() { + var updateCmd = &cobra.Command{ + Use: "update", + Short: "Check latest Update", + Long: core.Banner(), + RunE: runUpdate, + } + updateCmd.Flags().String("meta", "", "Custom MetaData URL") + updateCmd.Flags().Bool("F", false, "Shortcut for force update and clean old data at the same time") + updateCmd.Flags().BoolVar(&options.Update.ForceUpdate, "force", false, "Force update") + updateCmd.Flags().BoolVar(&options.Update.CleanOldData, "clean", false, "Clean up old Data") + updateCmd.Flags().BoolVar(&options.Update.VulnUpdate, "vuln", false, "Update Vulnerability Database only") + // generate update meta data + updateCmd.Flags().StringVar(&options.Update.UpdateURL, "update-url", "", "The script URL to download update") + updateCmd.Flags().StringVar(&options.Update.GenerateMeta, "gen", "", "Generate metadata for update") + RootCmd.AddCommand(updateCmd) + updateCmd.PreRun = func(cmd *cobra.Command, args []string) { + if options.FullHelp { + cmd.Help() + os.Exit(0) + } + } +} + +func runUpdate(cmd *cobra.Command, _ []string) error { + meta, _ := cmd.Flags().GetString("meta") + if meta != "" { + options.Update.MetaDataURL = meta + } + + forcedUpdateandClean, _ := cmd.Flags().GetBool("F") + if forcedUpdateandClean { + options.Update.ForceUpdate = true + options.Update.CleanOldData = true + } + + if options.Update.GenerateMeta != "" { + core.GenerateMetaData(options) + return nil + } + + if options.Update.VulnUpdate { + core.UpdateVuln(options) + return nil + } + + var shouldUpdate bool + if options.Update.UpdateURL == "" { + options.Update.UpdateURL = core.GetUpdateURL(options) + } + + if options.Update.ForceUpdate { + shouldUpdate = true + utils.InforF("Force to Update latest release") + } else { + shouldUpdate = core.CheckUpdate(&options) + } + + if shouldUpdate { + err := core.RunUpdate(options) + if err != nil { + return err + } + } + + return nil +} diff --git a/cmd/usage.go b/cmd/usage.go new file mode 100644 index 0000000..e7a325e --- /dev/null +++ b/cmd/usage.go @@ -0,0 +1,313 @@ +package cmd + +import ( + "fmt" + + "github.com/fatih/color" + "github.com/j3ssie/osmedeus/core" + "github.com/j3ssie/osmedeus/libs" + "github.com/spf13/cobra" +) + +// RootUsage base help +func RootUsage() { + var h string + h += ScanUsage() + h += CloudUsage() + h += QueueUsage() + h += ReportUsage() + h += UtilsUsage() + fmt.Println(h) +} + +func ScanExmaples() string { + h := color.HiCyanString("Example Scan Commands:") + h += color.HiBlueString("\n ## Start a simple scan with default 'general' flow\n") + h += " osmedeus scan -t sample.com\n" + + h += color.HiBlueString("\n ## Start a general scan but exclude some of the module\n") + h += " osmedeus scan -t sample.com -x screenshot -x spider\n" + + h += color.HiBlueString("\n ## Start a scan directly with a module with inputs as a list of http domains like this https://sub.example.com\n") + h += " osmedeus scan -m content-discovery -t http-file.txt\n" + + h += color.HiBlueString("\n ## Initiate the scan using a speed option other than the default setting\n") + h += " osmedeus scan -f vuln --tactic gently -t sample.com\n" + h += " osmedeus scan --threads-hold=10 -t sample.com\n" + h += " osmedeus scan -B 5 -t sample.com\n" + + h += color.HiBlueString("\n ## Start a simple scan with other flow\n") + h += " osmedeus scan -f vuln -t sample.com\n" + h += " osmedeus scan -f extensive -t sample.com -t another.com\n" + h += " osmedeus scan -f urls -t list-of-urls.txt\n" + + h += color.HiBlueString("\n ## Scan list of targets\n") + h += " osmedeus scan -T list_of_targets.txt\n" + h += " osmedeus scan -f vuln -T list-of-targets.txt\n" + + h += color.HiBlueString("\n ## Performing static vulnerability scan and secret scan on a git repo\n") + h += " osmedeus scan -m repo-scan -t https://github.com/j3ssie/sample-repo\n" + h += " osmedeus scan -m repo-scan -t /tmp/source-code-folder\n" + h += " osmedeus scan -m repo-scan -T list-of-repo.txt\n" + + h += color.HiBlueString("\n ## Scan for CIDR with file contains CIDR with the format '1.2.3.4/24'\n") + h += " osmedeus scan -f cidr -t list-of-ciders.txt\n" + h += " osmedeus scan -f cidr -t '1.2.3.4/24' # this will auto convert the single input to the file and run\n" + + h += color.HiBlueString("\n ## Directly run on vuln scan and directory scan on list of domains\n") + h += " osmedeus scan -f domains -t list-of-domains.txt\n" + h += " osmedeus scan -f vuln-and-dirb -t list-of-domains.txt\n" + + h += color.HiBlueString("\n ## Use a custom wordlist\n") + h += " osmedeus scan -t sample.com -p 'wordlists={{Data}}/wordlists/content/big.txt'\n" + + h += color.HiBlueString("\n ## Use a custom wordlist\n") + h += " cat list_of_targets.txt | osmedeus scan -c 2\n" + + h += color.HiBlueString("\n ## Start a normal scan and backup entire workflow folder to the backup folder\n") + h += " osmedeus scan --backup -f domains -t list-of-subdomains.txt\n" + + h += color.HiBlueString("\n ## Start the scan with chunk inputs to review the output way more much faster\n") + h += " osmedeus scan --chunk --chunk-parts 20 -f cidr -t list-of-100-cidr.txt\n" + + h += color.HiBlueString("\n ## Continuously run the scan on a target right after it finished\n") + h += " osmedeus utils cron --for --cmd 'osmedeus scan -t example.com'\n" + + h += color.HiBlueString("\n ## Backing up all workspaces\n") + h += " ls ~/workspaces-osmedeus | osmedeus report compress\n" + + h += "\n" + return h +} + +func ScanUsage() string { + h := ScanExmaples() + h += color.HiCyanString("\nScan Usage:\n") + h += " osmedeus scan -f [flowName] -t [target] \n" + h += " osmedeus scan -m [modulePath] -T [targetsFile] \n" + h += " osmedeus scan -f /path/to/flow.yaml -t [target] \n" + h += " osmedeus scan -m /path/to/module.yaml -t [target] --params 'port=9200'\n" + h += " osmedeus scan -m /path/to/module.yaml -t [target] -l /tmp/log.log\n" + h += " osmedeus scan --tactic aggressive -m module -t [target] \n" + h += " cat targets | osmedeus scan -f sample\n" + + h += color.HiCyanString("\nPractical Scan Usage:\n") + h += " osmedeus scan -T list_of_targets.txt -W custom_workspaces\n" + h += " osmedeus scan -t target.com -w workspace_name --debug\n" + h += " osmedeus scan -f general -t sample.com\n" + h += " osmedeus scan --tactic aggressive -f general -t sample.com\n" + h += " osmedeus scan -f extensive -t sample.com -t another.com\n" + h += " cat list_of_urls.txt | osmedeus scan -f urls\n" + h += " osmedeus scan --threads-hold=15 -f cidr -t 1.2.3.4/24\n" + h += " osmedeus scan -m ~/.osmedeus/core/workflow/test/dirbscan.yaml -t list_of_urls.txt\n" + h += " osmedeus scan --wfFolder ~/custom-workflow/ -f your-custom-workflow -t list_of_urls.txt\n" + h += " osmedeus scan --chunk --chunk-part 40 -c 2 -f cidr -t list-of-cidr.txt\n" + return h +} + +func UtilsUsage() string { + h := color.HiCyanString("\nUtilities Usage:\n") + h += color.HiBlueString(" ## Health Utility\n") + h += " osmedeus health \n" + h += " osmedeus health git\n" + h += " osmedeus health cloud\n" + h += " osmedeus version --json \n" + h += "\n" + + h += color.HiBlueString(" ## Set the base threads hold\n") + h += " osmedeus config set --threads-hold=10\n" + h += "\n" + + h += color.HiBlueString(" ## Update utilities\n") + h += " osmedeus update \n" + h += " osmedeus update --vuln\n" + h += " osmedeus update --force --clean \n" + h += "\n" + + h += color.HiBlueString(" ## Workflow utilities\n") + h += " osmedeus workflow list \n" + h += " osmedeus workflow view -f general\n" + h += "\n" + + h += color.HiBlueString(" ## Tmux utilities\n") + h += " osmedeus utils tmux ls \n" + h += " osmedeus utils tmux logs -A -l 10 \n" + h += "\n" + + h += color.HiBlueString(" ## Process utilities\n") + h += " osmedeus utils ps \n" + h += " osmedeus utils ps --proc 'jaeles' \n" + h += "\n" + + h += color.HiBlueString(" ## List all the sub proccess running by osmedeus\n") + h += " osmedeus utils ps --osm \n" + h += "\n" + + h += color.HiBlueString(" ## Kill all the sub proccess running by osmedeus\n") + h += " osmedeus utils ps --osm --kill \n" + h += "\n" + + h += color.HiBlueString(" ## Cron utilities\n") + h += " osmedeus utils cron --cmd 'osmdeus scan -t example.com' --sch 60\n" + h += " osmedeus utils cron --for --cmd 'osmedeus scan -t example.com'\n" + + return h +} + +func ConfigUsage() string { + h := color.HiCyanString("\nConfig Usage:\n") + h += " osmedeus config [action] [OPTIONS] \n" + h += " osmedeus config init -p https://github.com/j3ssie/osmedeus-plugins\n" + h += " osmedeus config --user newusser --pass newpassword\n" + h += " osmedeus config clean \n" + h += " osmedeus config delete -t woskapce \n" + h += " osmedeus config delete -w workspace_name \n" + h += " osmedeus config set --threads-hold=10 \n" + return h +} + +func QueueUsage() string { + h := color.HiCyanString("\nQueue Usage:\n") + h += " osmedeus queue -Q /tmp/queue-file.txt -c 2\n" + h += " osmedeus queue --add -t example.com -Q /tmp/queue-file.txt \n" + return h +} + +func CloudUsage() string { + h := color.HiCyanString("\nProvider Usage:\n") + h += " osmedeus provider wizard \n" + h += " osmedeus provider validate \n" + h += " osmedeus provider build --token xxx --rebuild --ic\n" + + h += " osmedeus provider health --debug \n" + h += " osmedeus provider health --for \n" + h += " osmedeus provider create --name 'sample' \n" + h += " osmedeus provider delete --id 34317111 --id 34317112 \n" + h += " osmedeus provider list \n" + + h += color.HiCyanString("\nCloud Usage:\n") + h += " osmedeus cloud -f [flowName] -t [target] \n" + h += " osmedeus cloud -f [flowName] -T [targetFile] --no-del\n" + h += " osmedeus cloud -m [modulePath] -t [target] \n" + h += " osmedeus cloud -c 5 -f [flowName] -T [targetsFile] \n" + h += " osmedeus cloud --token xxx -c 5 -f [flowName] -T [targetsFile] \n" + h += " osmedeus cloud --chunk -c 5 -f [flowName] -t [targetsFile] \n" + + return h +} + +func ReportUsage() string { + h := color.HiCyanString("\nReport Usage:\n") + h += " osmedeus report list\n" + h += " osmedeus report extract -t target.com.tar.gz\n" + h += " osmedeus report extract -t target.com.tar.gz --dest .\n" + h += " osmedeus report compress -t target.com\n" + h += " osmedeus report view --raw -t target.com\n" + h += " osmedeus report view --static -t target.com\n" + h += " osmedeus report view --static --ip 0 -t target.com\n" + return h +} + +func ServerUsage() string { + h := color.HiCyanString("\nServer Usage:\n") + h += " osmedeus server --port 5000\n" + h += " osmedeus server --disable-ssl\n" + h += " osmedeus server -A --disable-ssl\n" + return h +} + +func QueueHelp(cmd *cobra.Command, _ []string) { + fmt.Println(core.Banner()) + fmt.Println(cmd.UsageString()) + h := QueueUsage() + fmt.Println(h) + printDocs(cmd) +} + +// ScanHelp scan help message +func ScanHelp(cmd *cobra.Command, _ []string) { + fmt.Println(core.Banner()) + if options.FullHelp { + fmt.Println(cmd.UsageString()) + } + h := ScanUsage() + fmt.Println(h) + printDocs(cmd) +} + +// CloudHelp scan help message +func CloudHelp(cmd *cobra.Command, _ []string) { + fmt.Println(core.Banner()) + if options.FullHelp { + fmt.Println(cmd.UsageString()) + } + h := CloudUsage() + fmt.Println(h) + printDocs(cmd) +} + +// ServerHelp scan help message +func ServerHelp(cmd *cobra.Command, _ []string) { + fmt.Println(core.Banner()) + if options.FullHelp { + fmt.Println(cmd.UsageString()) + } + h := ServerUsage() + fmt.Println(h) + printDocs(cmd) +} + +// ConfigHelp config help message +func ConfigHelp(cmd *cobra.Command, _ []string) { + fmt.Println(core.Banner()) + if options.FullHelp { + fmt.Println(cmd.UsageString()) + } + h := ConfigUsage() + + fmt.Println(h) + printDocs(cmd) +} + +// UtilsHelp utils help message +func UtilsHelp(cmd *cobra.Command, _ []string) { + fmt.Println(core.Banner()) + if options.FullHelp { + fmt.Println(cmd.UsageString()) + } + h := UtilsUsage() + fmt.Println(h) + printDocs(cmd) +} + +// ReportHelp utils help message +func ReportHelp(cmd *cobra.Command, _ []string) { + fmt.Println(core.Banner()) + if options.FullHelp { + fmt.Println(cmd.UsageString()) + } + h := ReportUsage() + fmt.Println(h) + printDocs(cmd) +} + +// RootHelp print help message +func RootHelp(cmd *cobra.Command, _ []string) { + fmt.Println(core.Banner()) + if options.FullHelp { + fmt.Println(cmd.UsageString()) + } + RootUsage() + printDocs(cmd) +} + +func printDocs(cmd *cobra.Command) { + if !options.FullHelp { + if cmd.Use == libs.BINARY { + fmt.Printf("šŸ’” For full help message, please run: %s\n", color.GreenString("osmedeus --hh")) + } else { + fmt.Printf("šŸ’” For full help message, please run: %s or %s\n", color.GreenString("osmedeus --hh"), color.GreenString("osmedeus "+cmd.Use+" --hh")) + } + } + fmt.Printf("šŸ“– Documentation can be found here: %s\n", color.GreenString(libs.DOCS)) +} diff --git a/cmd/utils.go b/cmd/utils.go new file mode 100644 index 0000000..fe8541f --- /dev/null +++ b/cmd/utils.go @@ -0,0 +1,139 @@ +package cmd + +import ( + "fmt" + "os" + + "github.com/fatih/color" + "github.com/j3ssie/osmedeus/core" + "github.com/j3ssie/osmedeus/execution" + "github.com/j3ssie/osmedeus/libs" + "github.com/j3ssie/osmedeus/utils" + jsoniter "github.com/json-iterator/go" + "github.com/spf13/cobra" +) + +func init() { + var utilsCmd = &cobra.Command{ + Use: "utils", + Aliases: []string{"u", "util"}, + Short: "Utils to get some information from the system", + Long: core.Banner(), + RunE: runUtils, + } + + var psCmd = &cobra.Command{ + Use: "ps", + Short: "Utility to get information about running process", + Long: core.Banner(), + RunE: runPs, + } + psCmd.Flags().StringSlice("proc", []string{}, "Process name") + psCmd.Flags().Bool("kill", false, "Kill the all processes") + psCmd.Flags().Bool("osm", false, "Osmedeus related process only") + + var tmuxCmd = &cobra.Command{ + Use: "tmux", + Short: "Utility to get info from tmux", + Long: core.Banner(), + RunE: runTmux, + } + + tmuxCmd.Flags().BoolVarP(&options.Tmux.ApplyAll, "all", "A", false, "Apply for all tmux sessions") + tmuxCmd.Flags().StringVarP(&options.Tmux.SelectedWindow, "name", "n", "", "Apply for all tmux sessions") + tmuxCmd.Flags().StringVarP(&options.Tmux.Exclude, "exclude", "e", "server", "Exclude tmux session") + tmuxCmd.Flags().IntVarP(&options.Tmux.Limit, "limit", "l", 0, "Size of output content") + + var cronCmd = &cobra.Command{ + Use: "cron", + Short: "Utility to run command schedule", + Long: core.Banner(), + RunE: runCron, + } + cronCmd.Flags().IntVar(&options.Cron.Schedule, "sch", 0, "Number of minutes to schedule the job") + cronCmd.Flags().BoolVar(&options.Cron.Forever, "for", false, "Keep running forever right after the command done") + cronCmd.Flags().StringVar(&options.Cron.Command, "cmd", "", "Command to run") + + // add command + utilsCmd.PersistentFlags().BoolVar(&options.JsonOutput, "json", false, "Output as JSON") + utilsCmd.AddCommand(cronCmd) + utilsCmd.AddCommand(tmuxCmd) + utilsCmd.AddCommand(psCmd) + utilsCmd.SetHelpFunc(UtilsHelp) + RootCmd.AddCommand(utilsCmd) + + utilsCmd.PreRun = func(cmd *cobra.Command, args []string) { + if options.FullHelp { + cmd.Help() + os.Exit(0) + } + } +} + +func runUtils(_ *cobra.Command, _ []string) error { + fmt.Println(UtilsUsage()) + return nil +} + +func runPs(cmd *cobra.Command, _ []string) error { + processes, _ := cmd.Flags().GetStringSlice("process") + osmRelated, _ := cmd.Flags().GetBool("osm") + killProcess, _ := cmd.Flags().GetBool("kill") + + if osmRelated { + pids := execution.ListAllOsmedeusProcess() + for _, pid := range pids { + if killProcess { + utils.RunOSCommand(fmt.Sprintf("kill -9 %v", pid)) + } + } + return nil + } + + if len(processes) == 0 { + processes = append(processes, libs.BINARY) + } + + for _, process := range processes { + pss := execution.GetOsmProcess(process) + for _, ps := range pss { + if options.JsonOutput { + if data, err := jsoniter.MarshalToString(ps); err == nil { + fmt.Println(data) + } + continue + } + fmt.Printf("pid:%v %s %v\n", color.HiCyanString("%v", ps.PID), color.HiMagentaString("--"), ps.Command) + } + } + + return nil +} + +func runTmux(_ *cobra.Command, args []string) error { + tmux, err := core.InitTmux(options) + if err != nil { + return err + } + + for _, argument := range args { + switch argument { + case "l", "ls", "list": + tmux.ListTmux() + case "t", "log", "logs", "tai", "tail": + tmux.CatchSession() + } + } + return nil +} + +func runCron(_ *cobra.Command, _ []string) error { + if options.Cron.Schedule == 0 && options.Cron.Forever == false { + return fmt.Errorf("missing '--sche' flag") + } + if options.Cron.Forever { + options.Cron.Schedule = -1 + } + core.RunCron(options.Cron.Command, options.Cron.Schedule) + return nil +} diff --git a/cmd/version.go b/cmd/version.go new file mode 100644 index 0000000..7369feb --- /dev/null +++ b/cmd/version.go @@ -0,0 +1,176 @@ +package cmd + +import ( + "fmt" + "os" + "runtime" + "strings" + "time" + + "github.com/j3ssie/osmedeus/core" + "github.com/j3ssie/osmedeus/libs" + "github.com/j3ssie/osmedeus/utils" + jsoniter "github.com/json-iterator/go" + + //"github.com/mackerelio/go-osstat/cpu" + //"github.com/mackerelio/go-osstat/memory" + "github.com/spf13/cobra" + //"github.com/shirou/gopsutil/cpu" + "github.com/shirou/gopsutil/mem" +) + +func init() { + var versionCmd = &cobra.Command{ + Use: "version", + Short: "Show core version", + Long: core.Banner(), + RunE: runVersion, + } + versionCmd.Flags().BoolVarP(&options.Verbose, "verbose", "V", false, "Show stat info too") + versionCmd.Flags().BoolVar(&options.JsonOutput, "json", false, "Output as JSON") + RootCmd.AddCommand(versionCmd) +} + +func runVersion(_ *cobra.Command, _ []string) error { + if options.JsonOutput { + fmt.Println(PrintStat()) + return nil + } + + if !options.Verbose { + fmt.Printf("osmedeus %s by %s\n", libs.VERSION, libs.AUTHOR) + } else { + statInfo := PrintStat() + fmt.Printf("osmedeus %s by %s -- %s\n", libs.VERSION, libs.AUTHOR, statInfo) + } + return nil +} + +// StatData overview struct +type StatData struct { + CPU string `json:"cpu"` + Mem string `json:"mem"` + Name string `json:"name"` + Version string `json:"version"` +} + +// PrintStat print status +func PrintStat() string { + data := GetStat() + if data.CPU.Idle == 0.0 { + return strings.TrimSpace(utils.Emojif(":thought_balloon:", "not responding")) + } + var cpu string + cpuUsage := 100.0 - data.CPU.Idle + if cpuUsage <= 20.0 { + cpu = utils.Emojif(":green_circle:", " cpu: %0.2f", cpuUsage) + } else if (cpuUsage > 20.0) && (cpuUsage <= 50.0) { + cpu = utils.Emojif(":green_circle:", " cpu: %0.2f", cpuUsage) + } else if (cpuUsage > 50.0) && (cpuUsage <= 80.0) { + cpu = utils.Emojif(":orange_circle:", " cpu: %0.2f", cpuUsage) + } else { + cpu = utils.Emojif(":red_circle:", " cpu: %0.2f", cpuUsage) + } + + var mem string + memUsage := 100.0 - (data.Mem.Free/data.Mem.Total)*100 + if memUsage <= 20.0 { + mem = utils.Emojif(":green_circle:", " mem: %0.2f", memUsage) + } else if (memUsage > 20.0) && (memUsage <= 50.0) { + mem = utils.Emojif(":green_circle:", " mem: %0.2f", memUsage) + } else if (memUsage > 50.0) && (memUsage <= 80.0) { + mem = utils.Emojif(":orange_circle:", " mem: %0.2f", memUsage) + } else { + mem = utils.Emojif(":red_circle:", " mem: %0.2f", memUsage) + } + + name, _ := os.Hostname() + if options.JsonOutput { + stat := StatData{ + CPU: fmt.Sprintf("%v", cpuUsage), + Mem: fmt.Sprintf("%v", memUsage), + Name: name, + Version: fmt.Sprintf("osmedeus %s by %s", libs.VERSION, libs.AUTHOR), + } + if data, err := jsoniter.MarshalToString(stat); err == nil { + return data + } + } + + return fmt.Sprintf("%s: %12s - %s", name, strings.TrimSpace(cpu), strings.TrimSpace(mem)) +} + +type ServerStatData struct { + CPU struct { + System float64 + User float64 + Idle float64 + } + Mem struct { + Total float64 + Used float64 + Free float64 + Cached float64 + } +} + +// GetStat get stat data +// func GetStat() ServerStatData { +// var stat ServerStatData + +// before, err := cpu.Get() +// if err != nil { +// return stat +// } +// time.Sleep(time.Duration(1) * time.Second) +// after, err := cpu.Get() +// if err != nil { +// return stat +// } +// total := float64(after.Total - before.Total) +// stat.CPU.User = float64(after.User-before.User) / total * 100 +// stat.CPU.System = float64(after.System-before.System) / total * 100 +// stat.CPU.Idle = float64(after.Idle-before.Idle) / total * 100 +// // memory part +// memory, err := memory.Get() +// if err != nil { +// return stat +// } +// stat.Mem.Total = float64(memory.Total+memory.SwapTotal) / (1024 * 1024 * 1024) +// stat.Mem.Used = float64(memory.Used+memory.SwapUsed) / (1024 * 1024 * 1024) +// stat.Mem.Used = float64(memory.Used+memory.SwapUsed) / (1024 * 1024 * 1024) +// stat.Mem.Cached = float64(memory.Cached) / (1024 * 1024 * 1024) +// stat.Mem.Free = float64(memory.Free+memory.SwapFree) / (1024 * 1024 * 1024) +// return stat +// } + +// GetStat gets stat data +func GetStat() ServerStatData { + var stat ServerStatData + + var before runtime.MemStats + runtime.ReadMemStats(&before) + + time.Sleep(time.Duration(1) * time.Second) + + var after runtime.MemStats + runtime.ReadMemStats(&after) + + totalAlloc := float64(after.TotalAlloc - before.TotalAlloc) + sys := float64(after.Sys - before.Sys) + stat.CPU.User = totalAlloc / sys * 100 + stat.CPU.System = sys / sys * 100 + stat.CPU.Idle = 100 - stat.CPU.User - stat.CPU.System + + // Memory part + memory, err := mem.VirtualMemory() + if err != nil { + return stat + } + stat.Mem.Total = float64(memory.Total) / (1024 * 1024 * 1024) + stat.Mem.Used = float64(memory.Used) / (1024 * 1024 * 1024) + stat.Mem.Cached = float64(memory.Cached) / (1024 * 1024 * 1024) + stat.Mem.Free = float64(memory.Free) / (1024 * 1024 * 1024) + + return stat +} diff --git a/cmd/workflow.go b/cmd/workflow.go new file mode 100644 index 0000000..3eebf1b --- /dev/null +++ b/cmd/workflow.go @@ -0,0 +1,200 @@ +package cmd + +import ( + "fmt" + "os" + "strings" + + "github.com/fatih/color" + "github.com/j3ssie/osmedeus/core" + "github.com/j3ssie/osmedeus/utils" + "github.com/olekukonko/tablewriter" + "github.com/spf13/cobra" +) + +func init() { + + var workflowCmd = &cobra.Command{ + Use: "workflow", + Aliases: []string{"wf", "wl", "workflows", "wfs", "work", "works"}, + Short: "Listing all available workflows", + Long: core.Banner(), + } + + var workflowListCmd = &cobra.Command{ + Use: "list", + Aliases: []string{"ls"}, + Short: "Listing all available workflows", + Long: core.Banner(), + RunE: runWorkflow, + } + + var workflowViewCmd = &cobra.Command{ + Use: "view", + Aliases: []string{"viwe", "ve", "vi", "v"}, + Short: "View details of a workflow", + Long: core.Banner(), + RunE: runWorkflowView, + } + workflowViewCmd.Flags().Bool("all", false, "View all of the workflows") + + workflowCmd.AddCommand(workflowViewCmd) + workflowCmd.AddCommand(workflowListCmd) + workflowCmd.SetHelpFunc(UtilsHelp) + RootCmd.AddCommand(workflowCmd) + + workflowCmd.PreRun = func(cmd *cobra.Command, args []string) { + if options.FullHelp { + cmd.Help() + os.Exit(0) + } + } +} + +func runWorkflow(cmd *cobra.Command, _ []string) error { + listFlows() + fmt.Printf("\n------------------------------------------------------------\n") + listDefaultModules() + fmt.Printf("šŸ’” For full help message, please run: %s or %s\n", color.GreenString("osmedeus --hh"), color.GreenString("osmedeus scan --hh")) + return nil +} + +func runWorkflowView(cmd *cobra.Command, _ []string) error { + allFlows := core.ListFlow(options) + viewAll, _ := cmd.Flags().GetBool("all") + + if viewAll { + for _, flow := range allFlows { + err := viewWorkflow(flow) + if err != nil { + utils.ErrorF("Error viewing workflow: %v", err) + } + fmt.Printf("\n------------------------------------------------------------\n\n") + } + } else { + err := viewWorkflow(options.Scan.Flow) + if err != nil { + utils.ErrorF("Error viewing workflow: %v", err) + } + } + + h := color.HiCyanString("\nšŸ“„ Sample Usage:\n") + h += color.HiGreenString(" osmedeus scan -f %v", color.HiMagentaString(options.Scan.Flow)) + color.HiGreenString(" -t ") + color.HiMagentaString("[target]") + "\n" + h += color.HiGreenString(" osmedeus scan -f %v", color.HiMagentaString(options.Scan.Flow)) + color.HiGreenString(" -t ") + color.HiMagentaString("[target]") + color.HiGreenString(" -p ") + color.HiMagentaString("'enableSomething=false'") + "\n\n" + fmt.Printf(h) + + fmt.Printf("šŸ’” To list all of the workflows available, please run: %s\n", color.GreenString("osmedeus workflow ls")) + fmt.Printf("šŸ’” For full help message, please run: %s or %s\n", color.GreenString("osmedeus --hh"), color.GreenString("osmedeus scan --hh")) + return nil +} + +func viewWorkflow(workflowName string) error { + fmt.Printf("šŸ“– Viewing workflow detail: %v\n\n", color.GreenString(workflowName)) + allFlows := core.ListFlow(options) + flows := core.SelectFlow(workflowName, options) + if len(flows) == 0 { + utils.ErrorF("Flow not found in any of existing workflow [%v]", color.HiYellowString(strings.Join(allFlows, ", "))) + return fmt.Errorf("Flow %s not found", workflowName) + } + selectedWorkflow := flows[0] + + var content [][]string + parsedFlow, err := core.ParseFlow(selectedWorkflow) + if err != nil { + utils.ErrorF("Error parsing flow: %v", selectedWorkflow) + return err + } + + var totalSteps, totalModules int + parameters := make(map[string]string) + for _, param := range parsedFlow.Params { + for k, v := range param { + parameters[k] = v + } + } + + for _, routine := range parsedFlow.Routines { + // select module depend on the flow type + if routine.FlowFolder != "" { + parsedFlow.Type = routine.FlowFolder + } else { + parsedFlow.Type = parsedFlow.DefaultType + } + + modules := core.SelectModules(routine.Modules, options) + + // loop through all modules to get the parameters + for _, module := range modules { + parsedModule, err := core.ParseModules(module) + if err != nil || parsedModule.Name == "" { + continue + } + for _, param := range parsedModule.Params { + for k, v := range param { + + _, exist := parameters[k] + if parsedFlow.ForceParams && exist { + utils.DebugF("Skip override param: %v --> %v", k, v) + continue + } + parameters[k] = v + } + + } + totalSteps += len(parsedModule.Steps) + totalModules++ + } + } + + var toggleFlags, skippingFlags []string + for key, value := range parameters { + if value == "true" { + value = color.GreenString(value) + } else if value == "false" { + value = color.RedString(value) + } else { + + value = color.CyanString(value) + } + + if strings.HasPrefix(key, "enable") { + toggleFlags = append(toggleFlags, fmt.Sprintf("%v=%v", key, value)) + } + + if strings.HasPrefix(key, "skip") { + skippingFlags = append(skippingFlags, fmt.Sprintf("%v=%v", key, value)) + } + } + + workflowInfo := fmt.Sprintf("Name: %v", color.HiCyanString(parsedFlow.Name)) + ", " + fmt.Sprintf("Total Steps: %v", color.HiCyanString("%v", totalSteps)) + ", " + fmt.Sprintf("Total Modules: %v", color.HiCyanString("%v", totalModules)) + content = append(content, []string{ + "Workflow Information", workflowInfo, + }) + content = append(content, []string{ + "Description", parsedFlow.Desc, + }) + + content = append(content, []string{ + "Toggleable parameter", strings.Join(toggleFlags, ", "), + }) + + content = append(content, []string{ + "Skippable parameter", strings.Join(skippingFlags, ", "), + }) + + if parsedFlow.Usage != "" { + content = append(content, []string{ + "Examples Commands", strings.TrimSpace(parsedFlow.Usage), + }) + } + + table := tablewriter.NewWriter(os.Stdout) + table.SetRowLine(true) + table.SetBorders(tablewriter.Border{Left: true, Top: true, Right: true, Bottom: true}) + table.SetColWidth(120) + table.SetAutoWrapText(false) + table.AppendBulk(content) + table.Render() + + return nil +} diff --git a/core/backup.go b/core/backup.go new file mode 100644 index 0000000..2f422fb --- /dev/null +++ b/core/backup.go @@ -0,0 +1,68 @@ +package core + +import ( + "os" + "path" + "strings" + + "github.com/fatih/color" + "github.com/j3ssie/osmedeus/execution" + "github.com/j3ssie/osmedeus/libs" + "github.com/j3ssie/osmedeus/utils" +) + +// in workflow file +// Compress('{{Backup}}/{{Workspace}}.tar.gz', '{{Output}}') +// Decompress('{{Output}}', '{{Backup}}/{{Workspace}}.tar.gz') + +func (r *Runner) BackupWorkspace() { + utils.InforF("Backing up the workspace: %v", r.Target["Workspace"]) + outputDir := r.Target["Output"] + dest := path.Join(r.Opt.Env.BackupFolder, r.Target["Workspace"]) + ".tar.gz" + if utils.FileExists(dest) { + os.Remove(dest) + } + + execution.Compress(dest, outputDir) + if utils.FileExists(dest) { + utils.GoodF("Backup workspace save at %s", color.HiMagentaString(dest)) + } +} + +func CompressWorkspace(target string, opt libs.Options) { + utils.InforF("Backing up the workspace: %v", color.HiCyanString(target)) + outputDir := path.Join(opt.Env.WorkspacesFolder, target) + if utils.FolderLength(outputDir) == 0 { + utils.ErrorF("Workspace is empty: %s", outputDir) + return + } + + dest := path.Join(opt.Env.BackupFolder, target) + ".tar.gz" + if utils.FileExists(dest) { + os.Remove(dest) + } + + execution.Compress(dest, outputDir) + if utils.FileExists(dest) { + utils.InforF("The workspace has been backed up and saved in %s", color.HiMagentaString(dest)) + } +} + +func ExtractBackup(src string, opt libs.Options) { + if !utils.FileExists(src) { + utils.ErrorF("Backup file not found: %s", src) + return + } + + target := strings.ReplaceAll(path.Base(src), ".tar.gz", "") + dest := path.Join(opt.Report.ExtractFolder, target) + if !strings.HasSuffix(dest, "/") { + dest += "/" + } + + if utils.FolderExists(dest) { + utils.MakeDir(dest) + } + execution.Decompress(dest, src) + utils.GoodF("Extracting the %v to %s", color.HiCyanString(target), color.HiMagentaString(dest)) +} diff --git a/core/banner.go b/core/banner.go new file mode 100644 index 0000000..95e0ff5 --- /dev/null +++ b/core/banner.go @@ -0,0 +1,63 @@ +package core + +import ( + "github.com/fatih/color" + "github.com/j3ssie/osmedeus/libs" +) + +// Banner print ascii banner +func Banner() string { + version := color.HiWhiteString(libs.VERSION) + author := color.MagentaString(libs.AUTHOR) + //W := color.HiWhiteString(``) + b := color.GreenString(``) + + b += color.GreenString(` + + .;1tfLCL1, + .,,..;i;f0G; + ,:,tCC. ... + ;i:fCL,1LLtf1i;, + .,::tCL1LC1::;, .,, + ;1:tCL,tLt,1: + ,::tLf, 1Lf;::. + .ii:tLt. .1Lf;i1. + ,:;tf1 1ft;:: + .1;:tf1 `) + color.HiWhiteString(` ,i1t1, `) + color.GreenString(` ift;;1, + ,i:t;f.`) + color.HiWhiteString(` ,LLffLL: `) + color.GreenString(` tft;i: + .;:;fff `) + color.HiWhiteString(` .LCLLLf,`) + color.GreenString(` 1ffi:;. + :fi;Lff1. `) + color.HiWhiteString(`,;;:`) + color.GreenString(` ifffi;f; + .:::tCLLfi:,,:ifLfLt::;. + ,11:1CCCCCLLLLLLf1;1t: + .it;:;1fLLLLfft1;:;ti. + ,:;::;;;;;;;;;;, + .,::::::::,. + `) + + // + // + //b += "\n\t" + color.GreenString(` @@@@@@`) + //b += "\n\t" + color.GreenString(` .@@' '@@.`) + //b += "\n\t" + color.GreenString(` :@ @:`) + //b += "\n\t" + color.GreenString(` :@ %v:@`, W) + color.GreenString(` @:`) + //b += "\n\t" + color.GreenString(` :@ %v:@`, W) + color.GreenString(` @:`) + // + //b += "\n\t" + color.GreenString(` :@ @:`) + //b += "\n\t" + color.GreenString(` '@@. .@@'`) + //b += "\n\t" + color.GreenString(` @@@@@@`) + //b += "\n\t" + color.GreenString(` @@`) + //b += "\n\t" + color.HiCyanString(` @ `) + color.GreenString(`@@`) + color.HiCyanString(` @`) + //b += "\n\t" + color.HiWhiteString(` +@@`) + color.GreenString(` @@ `) + color.HiWhiteString(` @@+`) + //b += "\n\t" + color.GreenString(` @@:@#@,@@,@#@:@@`) + //b += "\n\t" + color.GreenString(` ;@+@@'#@@@@#'@@+@;`) + //b += "\n\t" + color.GreenString(` @+ #@@ @@ @@# +@`) + //b += "\n\t" + color.GreenString(` @@ @+'@@@@@@'+@ @@`) + //b += "\n\t" + color.GreenString(` @. @ ;@@; @ .@`) + //b += "\n\t" + color.BlueString(` #@ '@ @; @#`) + + b += "\n\n\t" + color.GreenString(` Osmedeus Next Generation %v`, version) + color.GreenString(` by %v`, author) + b += "\n\n" + color.HiCyanString(` %s`, libs.DESC) + "\n" + b += "\n" + color.HiWhiteString(` ĀÆ\_(惄)_/ĀÆ`) + "\n\n" + color.Unset() + return b +} diff --git a/core/config.go b/core/config.go new file mode 100644 index 0000000..f86ffac --- /dev/null +++ b/core/config.go @@ -0,0 +1,382 @@ +package core + +import ( + "fmt" + "os" + "path" + "path/filepath" + "runtime" + "strings" + + "github.com/fatih/color" + "github.com/j3ssie/osmedeus/libs" + "github.com/j3ssie/osmedeus/provider" + "github.com/j3ssie/osmedeus/utils" + "github.com/spf13/cast" + "github.com/spf13/viper" +) + +var v *viper.Viper + +// InitConfig Init the config +func InitConfig(options *libs.Options) error { + // ~/.osmedeus + RootFolder := filepath.Dir(utils.NormalizePath(options.ConfigFile)) + if !utils.FolderExists(RootFolder) { + if err := os.MkdirAll(RootFolder, 0750); err != nil { + return err + } + } + + // Base folder + // ~/osmedeus-base + BaseFolder := utils.NormalizePath(options.Env.BaseFolder) + if !utils.FolderExists(BaseFolder) { + fmt.Printf("%v Base folder not found at: %v\n", color.RedString("[Panic]"), color.HiGreenString(BaseFolder)) + fmt.Printf(color.HiYellowString("[!]")+" Consider running the installation script first: %v\n", color.HiGreenString("bash <(curl -fsSL %v)", libs.INSTALL)) + fmt.Printf(color.HiYellowString("[!]")+" Or better visit the installation guide at: %v\n", color.HiMagentaString("https://docs.osmedeus.org/installation/")) + os.Exit(-1) + } + + // load all the tokens + options.TokenConfigFile = path.Join(BaseFolder, "token/osm-var.yaml") + if !utils.FolderExists(path.Dir(options.TokenConfigFile)) { + utils.MakeDir(path.Dir(options.TokenConfigFile)) + } + + options.Env.WorkspacesFolder = utils.NormalizePath(options.Env.WorkspacesFolder) + if !utils.FolderExists(options.Env.WorkspacesFolder) { + utils.MakeDir(options.Env.WorkspacesFolder) + } + + // init config + options.ConfigFile = utils.NormalizePath(options.ConfigFile) + v = viper.New() + v.SetConfigName("config") + v.SetConfigType("yaml") + v.AddConfigPath(options.ConfigFile) + v.AddConfigPath(path.Dir(options.ConfigFile)) + + err := v.ReadInConfig() + if err != nil { + secret := utils.GenHash(utils.RandomString(8) + utils.GetTS())[:32] // only 32 char + prefix := secret[len(secret)-20 : len(secret)-1] + + // set some default config if config file doesn't exist + v.SetDefault("Server", map[string]string{ + "bind": "0.0.0.0:8000", + "cors": "*", + "secret": secret, + "prefix": prefix, + "ui": path.Join(RootFolder, "server/ui"), + "cert_file": path.Join(RootFolder, "server/ssl/cert.pem"), + "key_file": path.Join(RootFolder, "server/ssl/key.pem"), + "master_pass": "", + }) + + v.SetDefault("Tactic", map[string]any{ + "default": runtime.NumCPU() * 8, // 8,16,32,64 + "aggressive": runtime.NumCPU() * 16, + "gently": runtime.NumCPU() * 2, + }) + + v.SetDefault("Mics", map[string]string{ + "docs": utils.GetOSEnv("OSM_DOCS", libs.DOCS), + }) + + // DB connection config + dbPath := utils.NormalizePath(path.Join(RootFolder, "sqlite.db")) + v.SetDefault("Database", map[string]string{ + "db_host": utils.GetOSEnv("DB_HOST", "127.0.0.1"), + "db_port": utils.GetOSEnv("DB_PORT", "3306"), + "db_name": utils.GetOSEnv("DB_NAME", "osm-core"), + "db_user": utils.GetOSEnv("DB_USER", "root"), + "db_pass": utils.GetOSEnv("DB_PASS", ""), + // default will be file system + "db_path": utils.GetOSEnv("DB_PATH", dbPath), + // sqlite or mysql + "db_type": utils.GetOSEnv("DB_TYPE", "filesystem"), + }) + + // default user + password := utils.GenHash(utils.GetTS())[:15] + v.SetDefault("Client", map[string]string{ + "username": "osmedeus", + "password": password, + "jwt": "", + "dest": "http://127.0.0.1:8000", + }) + + v.SetDefault("Environments", map[string]string{ + // RootFolder --> ~/.osmedeus/ + "storages": path.Join(RootFolder, "storages"), + "backups": path.Join(RootFolder, "backups"), + "provider_config": path.Join(RootFolder, "provider"), + "instances": path.Join(RootFolder, "instances"), + + // store all the result + "workspaces": options.Env.WorkspacesFolder, + + // this update occasionally + // BaseFolder --> ~/osmedeus-base/ + "workflows": path.Join(BaseFolder, "workflow"), + "binaries": path.Join(BaseFolder, "binaries"), + "data": path.Join(BaseFolder, "data"), + "cloud_config": path.Join(BaseFolder, "cloud"), + }) + + if err := v.WriteConfigAs(options.ConfigFile); err != nil { + utils.ErrorF("Error writing config file: %s", err) + } + + utils.InforF("Created a new configuration file at %s", color.HiCyanString(options.ConfigFile)) + } + + if isWritable, _ := utils.IsWritable(options.ConfigFile); isWritable { + utils.ErrorF("config file does not writable: %v", color.HiCyanString(options.ConfigFile)) + utils.BlockF("fatal", "Make sure you are login as 'root user' if your installation done via root user") + os.Exit(-1) + } + SetupOSEnv(options) + return nil +} + +func LoadConfig(options *libs.Options) *viper.Viper { + v = viper.New() + v.SetConfigName("config") + v.SetConfigType("yaml") + v.AddConfigPath(options.ConfigFile) + v.AddConfigPath(path.Dir(options.ConfigFile)) + + if err := v.ReadInConfig(); err != nil { + utils.ErrorF("Error reading config file, %s", err) + } + return v +} + +func ParsingConfig(options *libs.Options) { + v = LoadConfig(options) + GetEnv(options) + GetServer(options) + GetClient(options) + SetupOpt(options) + // get the config for cloud provider + GetCloud(options) +} + +// GetEnv get environment options +func GetEnv(options *libs.Options) { + envs := v.GetStringMapString("Environments") + + options.Env.BinariesFolder = utils.NormalizePath(envs["binaries"]) + utils.MakeDir(options.Env.BinariesFolder) + + if options.Env.WorkFlowsFolder != "" { + options.Env.WorkFlowsFolder = utils.NormalizePath(options.Env.WorkFlowsFolder) + } + + options.Env.DataFolder = utils.NormalizePath(envs["data"]) + utils.MakeDir(options.Env.DataFolder) + // ose folder + options.Env.OseFolder = path.Join(options.Env.BaseFolder, "ose") + utils.MakeDir(options.Env.DataFolder) + options.Env.BackupFolder = utils.NormalizePath(envs["backups"]) + utils.MakeDir(options.Env.BackupFolder) + options.Env.UIFolder = path.Join(options.Env.BaseFolder, "ui") + + // local data + options.Env.StoragesFolder = utils.NormalizePath(envs["storages"]) + utils.MakeDir(options.Env.StoragesFolder) + options.Env.WorkspacesFolder = utils.NormalizePath(envs["workspaces"]) + utils.MakeDir(options.Env.WorkspacesFolder) + + customWorkflow := utils.GetOSEnv("CUSTOM_OSM_WORKFLOW", "CUSTOM_OSM_WORKFLOW") + if customWorkflow != "CUSTOM_OSM_WORKFLOW" && options.Env.WorkFlowsFolder == "" { + options.Env.WorkFlowsFolder = utils.NormalizePath(customWorkflow) + } + + if options.Env.WorkFlowsFolder == "" { + options.Env.WorkFlowsFolder = utils.NormalizePath(envs["workflows"]) + } + + // @NOTE: well of course you can rebuild the core engine binary to bypass this check + // However, the premium package primarily focuses on exclusive workflow and specialized wordlists. + // see more about it here: https://docs.osmedeus.org/faq/#premium-package-related-questions + // and https://docs.osmedeus.org/premium/ + if utils.FileExists(path.Join(options.Env.WorkFlowsFolder, "premium.md")) { + options.PremiumPackage = true + } + + // cloud stuff + + // ~/.osmedeus/providers/ + options.Env.ProviderFolder = utils.NormalizePath(envs["provider_config"]) + options.Env.InstancesFolder = utils.NormalizePath(envs["instances"]) + if options.Env.InstancesFolder == "" { + options.Env.InstancesFolder = utils.NormalizePath(path.Join(options.Env.RootFolder, "instances")) + } + utils.MakeDir(options.Env.ProviderFolder) + utils.MakeDir(options.Env.InstancesFolder) + + // ~/osmedeus-base/clouds/ + options.Env.CloudConfigFolder = utils.NormalizePath(envs["cloud_config"]) +} + +// SetupOpt get storage repos +func SetupOpt(options *libs.Options) { + // auto append PATH with Plugin folder + osPATH := utils.GetOSEnv("PATH", "PATH") + if !strings.Contains(osPATH, options.Env.BinariesFolder) { + utils.DebugF("Append $PATH with: %s", options.Env.BinariesFolder) + os.Setenv("PATH", fmt.Sprintf("%s:%s", osPATH, strings.TrimRight(options.Env.BinariesFolder, "/"))) + } + + tactics := v.GetStringMap("Tactic") + options.Tactics = strings.ToLower(options.Tactics) + options.ThreadsHold.Default = cast.ToInt(tactics["default"]) + options.ThreadsHold.Aggressive = cast.ToInt(tactics["aggressive"]) + options.ThreadsHold.Gently = cast.ToInt(tactics["gently"]) + + // try to autocorrect the tactic name + switch options.Tactics { + case "aggressive", "agg", "aggr", "aggrsive": + options.Tactics = "aggressive" + case "gently", "gen", "gent", "gentl": + options.Tactics = "gently" + } + defaultThreadsHold := cast.ToInt(tactics[options.Tactics]) + + if defaultThreadsHold == 0 { + utils.ErrorF("tactic %s not found, switching to the %v one", color.HiRedString(options.Tactics), color.HiYellowString("default")) + defaultThreadsHold = cast.ToInt(tactics["default"]) + options.Tactics = "default" + // in case you're still using the old config + if defaultThreadsHold == 0 { + defaultThreadsHold = 4 + } + } + + // override if you put --threads-hold flag + if options.Threads == 0 { + options.Threads = defaultThreadsHold + } + + /* some special conditions below */ + + // change {{.Storage}} from ~/.osmedeus/storages to ~/.osmedeus/destorages + if options.EnableDeStorage { + utils.DebugF("Dedicated Storage Enabled") + options.Env.StoragesFolder = options.Git.DeStorage + if !utils.FolderExists(options.Env.StoragesFolder) { + utils.MakeDir(options.Env.StoragesFolder) + } + } +} + +func GetCloud(options *libs.Options) { + if !options.PremiumPackage { + return + } + + // ~/osemedeus-base/cloud/provider.yaml + cloudConfigFile := path.Join(options.Env.CloudConfigFolder, "provider.yaml") + + options.CloudConfigFile = cloudConfigFile + utils.DebugF("Parsing cloud config from: %s", color.HiCyanString(options.CloudConfigFile)) + providerConfigs, err := provider.ParseProvider(options.CloudConfigFile) + + if err != nil { + utils.InforF("šŸ’” You can start the wizard with the command: %s", color.HiCyanString("%s provider wizard", libs.BINARY)) + } + + options.Cloud.BuildRepo = providerConfigs.Builder.BuildRepo + options.Cloud.SecretKey = utils.NormalizePath(providerConfigs.Builder.SecretKey) + options.Cloud.PublicKey = utils.NormalizePath(providerConfigs.Builder.PublicKey) + if options.Cloud.SecretKey == "" { + options.Cloud.SecretKey = path.Join(options.Env.CloudConfigFolder, "ssh/cloud") + options.Cloud.PublicKey = path.Join(options.Env.CloudConfigFolder, "ssh/cloud.pub") + } + + // check SSH Keys + if !utils.FileExists(options.Cloud.SecretKey) { + keysDir := path.Dir(options.Cloud.SecretKey) + os.RemoveAll(keysDir) + utils.MakeDir(keysDir) + + utils.InforF("Generate SSH Key at: %v", options.Cloud.SecretKey) + var err error + _, err = utils.RunCommandWithErr(fmt.Sprintf(`ssh-keygen -t ed25519 -f %s -q -N ''`, options.Cloud.SecretKey)) + if err != nil { + color.Red("[-] error generated SSH Key for cloud config at: %v", options.Cloud.SecretKey) + return + } + } + + if utils.FileExists(options.Cloud.SecretKey) { + utils.DebugF("Detected secret key: %v", options.Cloud.SecretKey) + utils.DebugF("Detected public key: %v", options.Cloud.PublicKey) + options.Cloud.SecretKeyContent = strings.TrimSpace(utils.GetFileContent(options.Cloud.SecretKey)) + options.Cloud.PublicKeyContent = strings.TrimSpace(utils.GetFileContent(options.Cloud.PublicKey)) + } +} + +// GetServer get server options +func GetServer(options *libs.Options) { + server := v.GetStringMapString("Server") + + options.Server.Bind = server["bind"] + options.Server.Cors = server["cors"] + options.Server.JWTSecret = server["secret"] + options.Server.StaticPrefix = server["prefix"] + options.Server.UIPath = utils.NormalizePath(server["ui"]) + utils.MakeDir(path.Dir(options.Server.UIPath)) + + options.Server.MasterPassword = server["master_pass"] + options.Server.CertFile = utils.NormalizePath(server["cert_file"]) + options.Server.KeyFile = utils.NormalizePath(server["key_file"]) + utils.MakeDir(path.Dir(options.Server.CertFile)) + + db := v.GetStringMapString("Database") + + options.Server.DBPath = utils.NormalizePath(db["db_path"]) + options.Server.DBType = db["db_type"] + // this should be remote one + if options.Server.DBType == "mysql" { + options.Server.DBUser = db["db_user"] + options.Server.DBPass = db["db_pass"] + options.Server.DBHost = db["db_host"] + options.Server.DBPort = db["db_port"] + options.Server.DBName = db["db_name"] + + // ā€œuser:password@/dbname?charset=utf8&parseTime=True&loc=Localā€ + cred := fmt.Sprintf("%v:%v", options.Server.DBUser, options.Server.DBPass) + dest := fmt.Sprintf("%v:%v", options.Server.DBHost, options.Server.DBPort) + dbURL := fmt.Sprintf("%v@tcp(%v)/%v?charset=utf8&parseTime=True&loc=Local", cred, dest, options.Server.DBName) + options.Server.DBConnection = dbURL + } +} + +// GetClient get options for client +func GetClient(options *libs.Options) map[string]string { + client := v.GetStringMapString("Client") + options.Client.Username = client["username"] + options.Client.Password = client["password"] + return client +} + +func SetTactic(options *libs.Options) { + v = LoadConfig(options) + if err := v.ReadInConfig(); err != nil { + utils.ErrorF("Error reading config file, %s", err) + } + + baseThreads := options.Threads + utils.InforF("Set base threads to %v", color.HiCyanString("%v", baseThreads)) + v.Set("Tactic", map[string]any{ + "default": baseThreads, // 2,4,8,16 + "aggressive": baseThreads * 4, + "gently": int(baseThreads / 2), + }) + + v.WriteConfig() +} diff --git a/core/cron.go b/core/cron.go new file mode 100644 index 0000000..f3c155d --- /dev/null +++ b/core/cron.go @@ -0,0 +1,30 @@ +package core + +import ( + "github.com/fatih/color" + "github.com/j3ssie/osmedeus/utils" + "github.com/jasonlvhit/gocron" + "github.com/spf13/cast" +) + +func taskWithParams(cmd string) { + utils.InforF("Exec: %v", color.HiMagentaString(cmd)) + _, err := utils.RunCommandSteamOutput(cmd) + if err != nil { + utils.ErrorF("Error running command: %v", err) + } +} + +func RunCron(cmd string, schedule int) { + + if schedule == -1 { + utils.InforF("Run command forever: %v", cmd) + for { + taskWithParams(cmd) + } + } + + utils.InforF("Start cron job with %v seconds: %v", schedule, color.HiCyanString(cmd)) + gocron.Every(cast.ToUint64(schedule)).Minutes().Do(taskWithParams, cmd) + <-gocron.Start() +} diff --git a/core/db.go b/core/db.go new file mode 100644 index 0000000..597f924 --- /dev/null +++ b/core/db.go @@ -0,0 +1,229 @@ +package core + +import ( + "os" + "path" + "strings" + "time" + + "github.com/fatih/color" + "github.com/j3ssie/osmedeus/database" + "github.com/j3ssie/osmedeus/libs" + "github.com/j3ssie/osmedeus/utils" + jsoniter "github.com/json-iterator/go" + "github.com/robertkrimen/otto" +) + +func (r *Runner) LoadDBScripts() string { + var output string + + r.VM.Set(TotalSubdomain, func(call otto.FunctionCall) otto.Value { + length := utils.FileLength(call.Argument(0).String()) + r.TargetObj.TotalAssets = length + utils.InforF("Total subdomain found: %v", color.HiMagentaString("%v", length)) + return otto.Value{} + }) + + r.VM.Set(TotalDns, func(call otto.FunctionCall) otto.Value { + length := utils.FileLength(call.Argument(0).String()) + r.TargetObj.TotalDns = length + utils.InforF("Total Dns: %v", color.HiMagentaString("%v", length)) + return otto.Value{} + }) + + r.VM.Set(TotalScreenShot, func(call otto.FunctionCall) otto.Value { + length := utils.FileLength(call.Argument(0).String()) + r.TargetObj.TotalScreenShot = length + utils.InforF("Total ScreenShot: %v", color.HiMagentaString("%v", length)) + return otto.Value{} + }) + + r.VM.Set(TotalTech, func(call otto.FunctionCall) otto.Value { + length := utils.FileLength(call.Argument(0).String()) + r.TargetObj.TotalTech = length + utils.InforF("Total Tech: %v", color.HiMagentaString("%v", length)) + return otto.Value{} + }) + + r.VM.Set(TotalVulnerability, func(call otto.FunctionCall) otto.Value { + data := utils.ReadingFileUnique(call.Argument(0).String()) + var length int + for _, line := range data { + if !strings.Contains(line, "-info") { + length += 1 + } + } + r.TargetObj.TotalVulnerability = length + utils.InforF("Total Vulnerability: %v", color.HiMagentaString("%v", length)) + return otto.Value{} + }) + + r.VM.Set(TotalArchive, func(call otto.FunctionCall) otto.Value { + length := utils.FileLength(call.Argument(0).String()) + r.TargetObj.TotalArchive = length + utils.InforF("Total Archive: %v", color.HiMagentaString("%v", length)) + return otto.Value{} + }) + + r.VM.Set(TotalLink, func(call otto.FunctionCall) otto.Value { + length := utils.FileLength(call.Argument(0).String()) + r.TargetObj.TotalLink = length + utils.InforF("Total Link: %v", color.HiMagentaString("%v", length)) + return otto.Value{} + }) + + r.VM.Set(TotalDirb, func(call otto.FunctionCall) otto.Value { + length := utils.FileLength(call.Argument(0).String()) + r.TargetObj.TotalDirb = length + utils.InforF("Total Dirb: %v", color.HiMagentaString("%v", length)) + return otto.Value{} + }) + + r.VM.Set(CreateReport, func(call otto.FunctionCall) otto.Value { + args := call.ArgumentList + reportPath := args[0].String() + if utils.FileExists(reportPath) { + return otto.Value{} + } + moduleName := "inline" + if len(args) > 1 { + moduleName = args[1].String() + } + reportItem := database.Report{ + ReportPath: reportPath, + Module: moduleName, + } + r.TargetObj.Reports = append(r.TargetObj.Reports, reportItem) + return otto.Value{} + }) + + return output +} + +func (r *Runner) DBNewTarget() { + r.TargetObj = database.Target{ + InputName: r.Input, + Workspace: r.Workspace, + InputType: r.InputType, + } + + r.DBRuntimeUpdate() +} + +func (r *Runner) DBNewScan() { + r.ScanObj = database.Scan{ + TaskType: r.RoutineType, + TaskName: path.Base(r.RoutineName), + + TotalSteps: r.TotalSteps, + InputName: r.Input, + InputType: r.InputType, + + MarkDownSunmmary: path.Join(r.WorkspaceFolder, "summary.md"), + MarkDownReport: path.Join(r.WorkspaceFolder, "summary.html"), + + LogFile: r.Opt.LogFile, + Target: r.TargetObj, + ProcessID: os.Getpid(), + IsRunning: true, + IsDone: false, + IsPrepared: true, + IsStarted: true, + } + + if r.RunnerType == "cloud" { + r.ScanObj.IsCloud = true + } + + if r.Opt.Scan.RemoteCall { + r.ScanObj.IsCloud = true + } + + r.ScanObj.CreatedAt = time.Now() + + if runtimeData, err := jsoniter.MarshalToString(r.ScanObj); err == nil { + utils.WriteToFile(r.RuntimeFile, runtimeData) + } + +} + +func (r *Runner) DBUpdateScan() { + r.ScanObj.DoneStep = r.DoneStep + r.ScanObj.CurrentModule = r.CurrentModule + r.ScanObj.RunningTime = r.RunningTime + r.ScanObj.ProcessID = os.Getpid() + + if r.ScanObj.DoneStep == r.ScanObj.TotalSteps { + r.ScanObj.IsDone = true + r.ScanObj.IsRunning = false + } else { + r.ScanObj.IsRunning = true + r.ScanObj.IsDone = false + + } + + utils.DebugF("[DB] Finished %v steps in the %v module", color.HiCyanString("%v/%v", r.DoneStep, r.TotalSteps), r.CurrentModule) + r.DBRuntimeUpdate() +} + +func (r *Runner) DBDoneScan() { + r.ScanObj.CurrentModule = "done" + r.ScanObj.RunningTime = r.RunningTime + + r.ScanObj.DoneStep = r.TotalSteps + r.ScanObj.IsDone = true + r.ScanObj.IsRunning = false + r.ScanObj.IsStarted = false + r.ScanObj.UpdatedAt = time.Now() + + utils.DebugF("[DB] The scan has been completed: %v -- %v", color.HiCyanString(r.ScanObj.InputName), color.HiCyanString(r.ScanObj.TaskName)) + if runtimeData, err := jsoniter.MarshalToString(r.ScanObj); err == nil { + utils.WriteToFile(r.DoneFile, runtimeData) + utils.WriteToFile(r.RuntimeFile, runtimeData) + } + + if utils.FileExists(r.ScanObj.MarkDownReport) { + utils.InforF("Markdown summary has been generated at: %v", color.GreenString(r.ScanObj.MarkDownReport)) + } + if utils.FileExists(r.ScanObj.MarkDownSunmmary) { + utils.InforF("HTML summary has been generated at: %v", color.GreenString(r.ScanObj.MarkDownSunmmary)) + } +} + +func (r *Runner) DBRuntimeUpdate() { + r.ScanObj.UpdatedAt = time.Now() + r.ScanObj.Target = r.TargetObj + if runtimeData, err := jsoniter.MarshalToString(r.ScanObj); err == nil { + utils.WriteToFile(r.RuntimeFile, runtimeData) + } +} + +func (r *Runner) DBNewReports(module libs.Module) { + r.ScanObj.CurrentModule = r.CurrentModule + r.ScanObj.RunningTime = r.RunningTime + + var reports []string + reports = append(reports, module.Report.Final...) + reports = append(reports, module.Report.Noti...) + reports = append(reports, module.Report.Diff...) + + utils.DebugF("Updating %v report records", len(reports)) + for _, report := range reports { + reportType := "text" + if strings.HasSuffix(report, ".html") { + reportType = "html" + } + + reportObj := database.Report{ + ReportName: path.Base(report), + ModulePath: module.ModulePath, + Module: module.Name, + ReportPath: report, + ReportType: reportType, + } + + r.TargetObj.Reports = append(r.TargetObj.Reports, reportObj) + + } + r.ScanObj.Target = r.TargetObj +} diff --git a/core/external.go b/core/external.go new file mode 100644 index 0000000..733b5fa --- /dev/null +++ b/core/external.go @@ -0,0 +1,298 @@ +package core + +import ( + "fmt" + "path" + "time" + + "github.com/j3ssie/osmedeus/execution" + "github.com/j3ssie/osmedeus/utils" + "github.com/robertkrimen/otto" +) + +func (r *Runner) LoadExternalScripts() string { + var output string + vm := r.VM + + // special scripts + vm.Set(Cleaning, func(call otto.FunctionCall) otto.Value { + if r.Opt.NoClean { + utils.InforF("Disabled Cleaning") + return otto.Value{} + } + execution.Cleaning(call.Argument(0).String(), r.Reports) + return otto.Value{} + }) + + // scripts for cleaning modules + vm.Set(CleanAmass, func(call otto.FunctionCall) otto.Value { + src := call.Argument(0).String() + dest := call.Argument(1).String() + execution.CleanAmass(src, dest) + return otto.Value{} + }) + + vm.Set(CleanRustScan, func(call otto.FunctionCall) otto.Value { + src := call.Argument(0).String() + dest := call.Argument(1).String() + execution.CleanRustScan(src, dest) + return otto.Value{} + }) + + vm.Set(CleanGoBuster, func(call otto.FunctionCall) otto.Value { + src := call.Argument(0).String() + dest := call.Argument(1).String() + execution.CleanGoBuster(src, dest) + return otto.Value{} + }) + vm.Set(CleanMassdns, func(call otto.FunctionCall) otto.Value { + src := call.Argument(0).String() + dest := call.Argument(1).String() + execution.CleanMassdns(src, dest) + return otto.Value{} + }) + + vm.Set(CleanSWebanalyze, func(call otto.FunctionCall) otto.Value { + src := call.Argument(0).String() + dest := call.Argument(1).String() + execution.CleanSWebanalyze(src, dest) + return otto.Value{} + }) + vm.Set(CleanJSONDnsx, func(call otto.FunctionCall) otto.Value { + src := call.Argument(0).String() + dest := call.Argument(1).String() + execution.CleanJSONDnsx(src, dest) + return otto.Value{} + }) + + vm.Set(CleanJSONHttpx, func(call otto.FunctionCall) otto.Value { + src := call.Argument(0).String() + dest := call.Argument(1).String() + execution.CleanJSONHttpx(src, dest) + return otto.Value{} + }) + + // Deprecated + vm.Set(CleanWebanalyze, func(call otto.FunctionCall) otto.Value { + src := call.Argument(0).String() + dest := call.Argument(1).String() + args := call.ArgumentList + + techSum := path.Join(path.Dir(dest), fmt.Sprintf("tech-overview-%v.txt", r.Target["Workspace"])) + if len(args) > 3 { + techSum = args[2].String() + } + execution.CleanWebanalyze(src, dest, techSum) + return otto.Value{} + }) + + vm.Set(CleanArjun, func(call otto.FunctionCall) otto.Value { + // src mean folder contain arjun output + src := call.Argument(0).String() + dest := call.Argument(1).String() + execution.CleanArjun(src, dest) + return otto.Value{} + }) + + vm.Set(CleanFFUFJson, func(call otto.FunctionCall) otto.Value { + src := call.Argument(0).String() + dest := call.Argument(1).String() + execution.CleanFFUFJson(src, dest) + return otto.Value{} + }) + + vm.Set(GenNucleiReport, func(call otto.FunctionCall) otto.Value { + src := call.Argument(0).String() + dest := call.Argument(1).String() + args := call.ArgumentList + + templateFile := "" + if len(args) >= 3 { + templateFile = args[2].String() + } + execution.GenNucleiReport(r.Opt, src, dest, templateFile) + return otto.Value{} + }) + + return output +} + +func (r *Runner) LoadGitScripts() string { + var output string + vm := r.VM + options := r.Opt + + // Clone("git@xxx.git", "/tmp/dest") + vm.Set(Clone, func(call otto.FunctionCall) otto.Value { + execution.GitClone(call.Argument(0).String(), call.Argument(1).String(), false, options) + return otto.Value{} + }) + // like clone but delete the destination folder first + vm.Set(FClone, func(call otto.FunctionCall) otto.Value { + execution.GitClone(call.Argument(0).String(), call.Argument(1).String(), true, options) + return otto.Value{} + }) + + vm.Set(PushResult, func(call otto.FunctionCall) otto.Value { + for folder := range options.Storages { + execution.PullResult(folder, options) + time.Sleep(3 * time.Second) + execution.PullResult(folder, options) + commitMess := fmt.Sprintf("%v|%v|%v", options.Module.Name, options.Scan.ROptions["Workspace"], utils.GetCurrentDay()) + execution.PushResult(folder, commitMess, options) + } + return otto.Value{} + }) + // push result but specific folder + vm.Set(PushFolder, func(call otto.FunctionCall) otto.Value { + folder := call.Argument(0).String() + execution.PullResult(folder, options) + time.Sleep(3 * time.Second) + execution.PullResult(folder, options) + commitMess := fmt.Sprintf("%v|%v|%v", options.Module.Name, options.Scan.ROptions["Workspace"], utils.GetCurrentDay()) + execution.PushResult(folder, commitMess, options) + return otto.Value{} + }) + + // push result but specific folder + vm.Set(PullFolder, func(call otto.FunctionCall) otto.Value { + folder := call.Argument(0).String() + execution.PullResult(folder, options) + time.Sleep(3 * time.Second) + execution.PullResult(folder, options) + return otto.Value{} + }) + + vm.Set(DiffCompare, func(call otto.FunctionCall) otto.Value { + src := call.Argument(0).String() + dest := call.Argument(1).String() + output := call.Argument(2).String() + execution.DiffCompare(src, dest, output, options) + return otto.Value{} + }) + + vm.Set(GitDiff, func(call otto.FunctionCall) otto.Value { + args := call.ArgumentList + src := args[0].String() + output := call.Argument(1).String() + history := "1" + if len(args) < 2 { + history = call.Argument(2).String() + } + execution.GitDiff(src, output, history, options) + return otto.Value{} + }) + vm.Set(LoopGitDiff, func(call otto.FunctionCall) otto.Value { + args := call.ArgumentList + src := args[0].String() + output := call.Argument(1).String() + execution.LoopGitDiff(src, output, options) + return otto.Value{} + }) + + /* --- CDN S3 Bucket --- */ + + // UploadToS3("/tmp/src", "xxx/xxx") or UploadToS3("/tmp/src", "your-cdn.s3.ap-southeast-1.amazonaws.com") + vm.Set(UploadToS3, func(call otto.FunctionCall) otto.Value { + args := call.ArgumentList + bucket := "" + src := args[0].String() + if len(args) > 1 { + bucket = args[1].String() + } + + // if bucket is empty, use the default one + if bucket == "" { + bucket = options.Cdn.Bucket + } + + execution.UploadToS3(options, src, bucket) + return otto.Value{} + }) + + // DownloadFromS3("xxx", "/tmp/dest") + vm.Set(DownloadFromS3, func(call otto.FunctionCall) otto.Value { + args := call.ArgumentList + bucket := "" + src := args[0].String() + dest := args[1].String() + if len(args) > 2 { + bucket = args[2].String() + } + + // if bucket is empty, use the default one + if bucket == "" { + bucket = options.Cdn.Bucket + } + + execution.DownloadFromS3(options, src, dest, bucket) + return otto.Value{} + }) + + // DownloadFile("https://xxx.com", "/tmp/dest") + vm.Set(DownloadFile, func(call otto.FunctionCall) otto.Value { + args := call.ArgumentList + src := args[0].String() + output := args[1].String() + execution.DownloadFile(options, src, output) + return otto.Value{} + }) + + /* --- end CDN S3 Bucket --- */ + + /* --- Gitlab API --- */ + + // CreateRepo("repo-name") + // CreateRepo("repo-name", "tags") + vm.Set(CreateRepo, func(call otto.FunctionCall) otto.Value { + args := call.ArgumentList + repoName := args[0].String() + tags := "" + if len(args) > 1 { + tags = args[1].String() + } + execution.CreateGitlabRepo(repoName, tags, options) + return otto.Value{} + }) + + vm.Set(DeleteRepo, func(call otto.FunctionCall) otto.Value { + args := call.ArgumentList + repoName := args[0].String() + execution.DeleteRepo(repoName, 0, options) + return otto.Value{} + }) + vm.Set(DeleteRepoByPid, func(call otto.FunctionCall) otto.Value { + args := call.ArgumentList + pid, err := args[0].ToInteger() + if err != nil { + return otto.Value{} + } + execution.DeleteRepo("", int(pid), options) + return otto.Value{} + }) + vm.Set(ListProjects, func(call otto.FunctionCall) otto.Value { + args := call.ArgumentList + if len(args) > 0 { + uid, err := args[0].ToInteger() + if err == nil { + execution.ListProjects(int(uid), options) + } + return otto.Value{} + } + execution.ListProjects(0, options) + return otto.Value{} + }) + + /* --- end Gitlab API --- */ + + // GenMarkdownReport("markdown.md", "output.html") + vm.Set(GenMarkdownReport, func(call otto.FunctionCall) otto.Value { + args := call.ArgumentList + markdownFile := args[0].String() + outputHTML := args[1].String() + r.GenMarkdownReport(markdownFile, outputHTML) + return otto.Value{} + }) + + return output +} diff --git a/core/flow.go b/core/flow.go new file mode 100644 index 0000000..496f913 --- /dev/null +++ b/core/flow.go @@ -0,0 +1,222 @@ +package core + +import ( + "fmt" + "path" + "path/filepath" + "strings" + + "github.com/fatih/color" + "github.com/j3ssie/osmedeus/utils" + "github.com/thoas/go-funk" + + "github.com/j3ssie/osmedeus/libs" +) + +func ListAllFlowName(options libs.Options) (result []string) { + rawWorkflows := ListModules(options) + for _, item := range rawWorkflows { + result = append(result, strings.ReplaceAll(filepath.Base(item), ".yaml", "")) + } + return result +} + +func ListModuleName(options libs.Options) (result []string) { + rawResult := DefaultWorkflows(options) + for _, item := range rawResult { + result = append(result, strings.ReplaceAll(filepath.Base(item), ".yaml", "")) + } + return result +} + +// ListFlow list all available mode +func ListFlow(options libs.Options) (result []string) { + modePath := path.Join(options.Env.WorkFlowsFolder, "/*.yaml") + result, err := filepath.Glob(modePath) + if err != nil { + return result + } + return result +} + +// SelectFlow select flow to run +func SelectFlow(flowName string, options libs.Options) []string { + flows := ListFlow(options) + var selectedFlow []string + + // absolute path like -f customflows/general.yaml + if strings.HasSuffix(flowName, ".yaml") { + if utils.FileExists(flowName) { + selectedFlow = append(selectedFlow, flowName) + return selectedFlow + } + } + + // -f test + if !strings.Contains(flowName, ",") { + selectedFlow = append(selectedFlow, singleMode(flowName, flows)...) + } + + // -f test1,test2 + flowNames := strings.Split(flowName, ",") + for _, item := range flowNames { + selectedFlow = append(selectedFlow, singleMode(item, flows)...) + } + + // default custom flow folder + if !utils.FileExists(flowName) { + flowName = path.Join(options.Env.WorkFlowsFolder, "default-flows", flowName) + if utils.FileExists(flowName) { + selectedFlow = append(selectedFlow, flowName) + } else if utils.FileExists(flowName + ".yaml") { + flowName = flowName + ".yaml" + selectedFlow = append(selectedFlow, flowName) + } + } + + selectedFlow = funk.UniqString(selectedFlow) + return selectedFlow +} + +func singleMode(modeName string, modes []string) (selectedMode []string) { + for _, mode := range modes { + basemodeName := strings.TrimRight(strings.TrimRight(filepath.Base(mode), "yaml"), ".") + // select workflow file in workflow directory + if strings.ToLower(basemodeName) == strings.ToLower(modeName) { + selectedMode = append(selectedMode, mode) + } + } + return selectedMode +} + +// ListModules list all available module +func ListModules(options libs.Options) (modules []string) { + modePath := path.Join(options.Env.WorkFlowsFolder, "general/*.yaml") + if options.Flow.Type != "" { + modePath = path.Join(options.Env.WorkFlowsFolder, fmt.Sprintf("%v/*.yaml", options.Flow.Type)) + } + if strings.HasSuffix(options.Scan.Flow, ".yaml") { + if options.Flow.Type == "" { + options.Flow.Type = "general" + } + modePath = path.Join(path.Dir(options.Scan.Flow), options.Flow.Type) + "/*.yaml" + } + modules, err := filepath.Glob(modePath) + if err != nil { + return modules + } + return modules +} + +// SelectModules return list of modules name +func SelectModules(moduleNames []string, options libs.Options) []string { + if strings.Contains(options.Flow.Type, "{{.") { + options.Flow.Type = ResolveData(options.Flow.Type, options.Scan.ROptions) + } + modules := ListModules(options) + var selectedModules []string + for _, item := range moduleNames { + selectedModules = append(selectedModules, singleSelectModule(item, modules)...) + } + selectedModules = funk.UniqString(selectedModules) + + utils.DebugF("Select module name %v: %v", color.HiCyanString("%v", moduleNames), selectedModules) + return selectedModules +} + +func singleSelectModule(moduleName string, modules []string) (selectedModules []string) { + for _, module := range modules { + baseModuleName := strings.Trim(strings.TrimRight(filepath.Base(module), "yaml"), ".") + if strings.ToLower(baseModuleName) == strings.ToLower(moduleName) { + selectedModules = append(selectedModules, module) + } + } + return selectedModules +} + +// DefaultWorkflows select module from ~/.osmedeus/core/workflow/plugins/ +func DefaultWorkflows(options libs.Options) []string { + defaultModule := path.Join(options.Env.WorkFlowsFolder, "default-modules") + modePath := path.Join(defaultModule, "/*.yaml") + results, err := filepath.Glob(modePath) + if err != nil { + utils.ErrorF("No default module found in %v", defaultModule) + return []string{} + } + return results +} + +// DirectSelectModule select module from ~/osmedeus-base/workflow/default-modules +func DirectSelectModule(options libs.Options, moduleName string) string { + // got absolutely path + if utils.FileExists(moduleName) { + return moduleName + } + + // select in cloud folder first if we're running the cloud scan + // ~/.osmedeus/core/workflow/cloud-modules/ + basePlugin := path.Join(options.Env.WorkFlowsFolder, "cloud-modules") + modulePath := path.Join(basePlugin, moduleName) + if utils.FileExists(modulePath) { + utils.DebugF("Load module path: %v", modulePath) + return modulePath + } + + modulePath = path.Join(basePlugin, moduleName+".yaml") + if utils.FileExists(modulePath) { + utils.DebugF("Load module path: %v", modulePath) + return modulePath + } + + // ~/.osmedeus/core/workflow/default-modules/ + basePlugin = path.Join(options.Env.WorkFlowsFolder, "default-modules") + modulePath = path.Join(basePlugin, moduleName) + utils.DebugF("Load module path: %v", modulePath) + if utils.FileExists(modulePath) { + return modulePath + } + + modulePath = path.Join(basePlugin, moduleName+".yaml") + utils.DebugF("Load module path: %v", modulePath) + if utils.FileExists(modulePath) { + return modulePath + } + utils.DebugF("No plugin found with: %v", moduleName) + return "" +} + +// ListScripts list all available mode +func ListScripts(options libs.Options) (result []string) { + modePath := path.Join(options.Env.OseFolder, "/*.js") + result, err := filepath.Glob(modePath) + if err != nil { + return result + } + + modePath = path.Join(options.Env.OseFolder, "/*/*.js") + DepthResult, err := filepath.Glob(modePath) + if err == nil { + result = append(result, DepthResult...) + } + return result +} + +func SelectScript(scriptName string, options libs.Options) string { + scripts := ListScripts(options) + for _, script := range scripts { + if strings.Contains(scriptName, "/") { + if strings.HasSuffix(script, scriptName) || strings.HasSuffix(script, scriptName+".js") { + return script + } + } + + compareName := path.Base(script) + if compareName == scriptName { + return script + } + if compareName == fmt.Sprintf("%s.js", scriptName) { + return script + } + } + return "" +} diff --git a/core/import.go b/core/import.go new file mode 100644 index 0000000..3da52e3 --- /dev/null +++ b/core/import.go @@ -0,0 +1,574 @@ +package core + +// func (r *Runner) LoadImportScripts() string { +// var output string + +// // DB scripts + +// r.VM.Set("ImportSubdomain", func(call otto.FunctionCall) otto.Value { +// src := call.Argument(0).String() +// r.ImportSubdomain(src) +// return otto.Value{} +// }) + +// r.VM.Set("ImportDns", func(call otto.FunctionCall) otto.Value { +// src := call.Argument(0).String() +// r.ImportDns(src) +// return otto.Value{} +// }) + +// r.VM.Set("ImportTech", func(call otto.FunctionCall) otto.Value { +// src := call.Argument(0).String() +// r.ImportTech(src) +// return otto.Value{} +// }) + +// r.VM.Set("ImportHTTPJson", func(call otto.FunctionCall) otto.Value { +// src := call.Argument(0).String() +// r.ImportHTTPJson(src) +// return otto.Value{} +// }) + +// r.VM.Set("ImportScreenShotJson", func(call otto.FunctionCall) otto.Value { +// src := call.Argument(0).String() +// r.ImportScreenShotJson(src) +// return otto.Value{} +// }) + +// r.VM.Set("ImportPortJson", func(call otto.FunctionCall) otto.Value { +// src := call.Argument(0).String() +// r.ImportPortJson(src) +// return otto.Value{} +// }) + +// r.VM.Set("ImportJaelesVulnJson", func(call otto.FunctionCall) otto.Value { +// src := call.Argument(0).String() +// r.ImportJaelesVulnJson(src) +// return otto.Value{} +// }) + +// r.VM.Set("ImportNucleiVulnJson", func(call otto.FunctionCall) otto.Value { +// src := call.Argument(0).String() +// r.ImportNucleiVulnJson(src) +// return otto.Value{} +// }) + +// r.VM.Set("ImportDirectoryJson", func(call otto.FunctionCall) otto.Value { +// src := call.Argument(0).String() +// r.ImportDirectoryJson(src) +// return otto.Value{} +// }) + +// r.VM.Set("ImportLinks", func(call otto.FunctionCall) otto.Value { +// src := call.Argument(0).String() +// r.ImportLinks(src) +// return otto.Value{} +// }) + +// r.VM.Set("ImportArchive", func(call otto.FunctionCall) otto.Value { +// src := call.Argument(0).String() +// r.ImportArchive(src) +// return otto.Value{} +// }) + +// r.VM.Set("ImportIPRange", func(call otto.FunctionCall) otto.Value { +// src := call.Argument(0).String() +// r.ImportIPRange(src) +// return otto.Value{} +// }) + +// r.VM.Set("ImportCred", func(call otto.FunctionCall) otto.Value { +// src := call.Argument(0).String() +// r.ImportCred(src) +// return otto.Value{} +// }) + +// r.VM.Set("ImportCert", func(call otto.FunctionCall) otto.Value { +// src := call.Argument(0).String() +// r.ImportCert(src) +// return otto.Value{} +// }) + +// r.VM.Set("ImportCloudBrute", func(call otto.FunctionCall) otto.Value { +// src := call.Argument(0).String() +// r.ImportCloudBrute(src) +// return otto.Value{} +// }) + +// r.VM.Set("SummaryTarget", func(call otto.FunctionCall) otto.Value { +// r.ScanObj.SummaryTarget() +// return otto.Value{} +// }) + +// return output +// } + +// func (r *Runner) ImportSubdomain(src string) { +// if !utils.FileExists(src) { +// utils.ErrorF("file not found: %v", src) +// return +// } + +// domains := utils.ReadingLines(src) +// var objs []database.Asset + +// for _, domain := range domains { +// domain = strings.TrimSpace(domain) +// if domain == "" { +// continue +// } + +// obj := database.Asset{ +// AssetValue: domain, +// //Dns: nil, +// HTTP: nil, +// Directory: nil, +// Vulnerability: nil, +// ScanRefer: r.ScanObj.ID, +// TargetRefer: r.ScanObj.TargetRefer, +// } +// objs = append(objs, obj) +// } +// database.ImportAssets(objs) +// } + +// func (r *Runner) ImportDns(src string) { +// if !utils.FileExists(src) { +// utils.ErrorF("file not found: %v", src) +// return +// } + +// content := utils.ReadingLines(src) +// for _, line := range content { +// line = strings.TrimSpace(line) +// if line == "" || !strings.Contains(line, " ") { +// continue +// } + +// raw := strings.Split(line, " ") +// domain := strings.Trim(raw[0], ".") +// dnsType := strings.TrimSpace(raw[1]) +// dnsValue := strings.Trim(raw[2], ".") + +// if strings.TrimSpace(domain) == "" || strings.TrimSpace(dnsValue) == "" { +// continue +// } + +// dnsChecksum := utils.GenHash(fmt.Sprintf("%s-%s-%s", domain, dnsType, dnsValue)) + +// obj := database.Dns{ +// Domain: domain, +// DnsType: dnsType, +// DnsValue: dnsValue, +// DnsChecksum: dnsChecksum, +// ScanRefer: r.ScanObj.ID, +// TargetRefer: r.ScanObj.TargetRefer, +// } +// obj.Create() +// } +// } + +// func (r *Runner) ImportTech(src string) { +// if !utils.FileExists(src) { +// utils.ErrorF("file not found: %v", src) +// return +// } + +// content := utils.ReadingLines(src) +// for _, line := range content { +// if !strings.Contains(line, ";;") { +// utils.ErrorF("Invalid format: %v", line) +// continue +// } + +// // data should be domain|%v;;techs|%v +// domain := strings.TrimPrefix(strings.Split(line, ";;")[0], "domain|") +// techs := strings.TrimPrefix(strings.Split(line, ";;")[1], "techs|") +// if strings.TrimSpace(techs) == "" { +// utils.ErrorF("Invalid format: %v", line) +// continue +// } + +// obj := database.Asset{ +// AssetValue: domain, +// Technology: techs, +// IsAlive: true, +// ScanRefer: r.ScanObj.ID, +// TargetRefer: r.ScanObj.TargetRefer, +// } + +// obj.UpdateTech() +// } +// } + +// func (r *Runner) ImportHTTPJson(src string) { +// if !utils.FileExists(src) { +// utils.ErrorF("file not found: %v", src) +// return +// } + +// content := utils.ReadingLines(src) + +// baseResult, _ := filepath.Abs(src) +// baseResult = path.Dir(baseResult) +// utils.DebugF("Set Base Dir for content: %v", baseResult) + +// for _, line := range content { +// if strings.TrimSpace(line) == "" { +// continue +// } +// jsonParsed, err := gabs.ParseJSON([]byte(line)) +// if err != nil { +// continue +// } + +// // {"url":"https://sso-na.tesla.com","title":"Blank Title","checksum":"527ef0b39a78caf74f54ca5b2ffb59bcfe688685","content_file":"overview/contents/https___sso-na.tesla.com.txt","status":"302","time":"0.046268348","length":"90","redirect":"https://teamchatgl.tesla.com/"} +// URL := jsonParsed.S("url").Data().(string) +// title := jsonParsed.S("title").Data().(string) +// checksum := jsonParsed.S("checksum").Data().(string) +// contentPath := jsonParsed.S("content_file").Data().(string) +// status := jsonParsed.S("status").Data().(string) +// length := jsonParsed.S("length").Data().(string) +// redirect := jsonParsed.S("redirect").Data().(string) + +// if !utils.FileExists(contentPath) { +// contentPath = path.Join(baseResult, contentPath) +// } + +// // base64 content +// data := "No-Content" +// if !strings.Contains(contentPath, "No-Content") { +// //utils.DebugF("Reading HTML content: %v", contentPath) +// data = utils.ImageAsBase64(contentPath) +// } + +// obj := database.HTTP{ +// URL: URL, +// Title: title, +// Checksum: checksum, +// StatusCode: cast.ToInt(status), +// ContentLength: cast.ToInt(length), +// HTTPContent: data, +// Redirect: redirect, +// ScanRefer: r.ScanObj.ID, +// TargetRefer: r.ScanObj.TargetRefer, +// } +// obj.CreateHTTP() +// } +// } + +// func (r *Runner) ImportScreenShotJson(src string) { +// if !utils.FileExists(src) { +// utils.ErrorF("file not found: %v", src) +// return +// } + +// baseResult, _ := filepath.Abs(src) +// baseResult = path.Dir(baseResult) +// content := utils.ReadingLines(src) +// utils.DebugF("Set Base Dir for screenshot: %v", baseResult) + +// for _, line := range content { +// if strings.TrimSpace(line) == "" { +// continue +// } +// jsonParsed, err := gabs.ParseJSON([]byte(line)) +// if err != nil { +// continue +// } + +// URL := jsonParsed.S("url").Data().(string) +// imgPath := jsonParsed.S("image").Data().(string) +// tech := jsonParsed.S("tech").Data().(string) +// if tech != "" { +// //domain, _ := utils.GetDomain(URL) +// utils.DebugF("more tech: %v", tech) +// //database.NewTech(wsObj, domain, tech) +// } + +// if !utils.FileExists(imgPath) { +// imgPath = path.Join(baseResult, imgPath) +// } +// imgData := utils.ImageAsBase64(imgPath) + +// //database.NewScreenshot(wsObj, URL, data) + +// obj := database.HTTP{ +// URL: URL, +// //Title: tittle, +// //Checksum: "", +// //StatusCode: 0, +// //ContentLength: 0, +// //HTTPContent: "", +// ScreenShotData: imgData, +// ScanRefer: r.ScanObj.ID, +// TargetRefer: r.ScanObj.TargetRefer, +// } +// obj.CreateScreenShot() +// } +// } + +// type PortObj struct { +// Protocol string +// PortID string +// State string +// Service struct { +// Name string +// Product string +// Cpe string +// } +// Script struct { +// ID string +// Output string +// } +// } + +// func (r *Runner) ImportPortJson(src string) { +// if !utils.FileExists(src) { +// utils.ErrorF("file not found: %v", src) +// return +// } +// content := utils.ReadingLines(src) + +// for _, line := range content { +// if strings.TrimSpace(line) == "" { +// continue +// } +// jsonParsed, err := gabs.ParseJSON([]byte(line)) +// if err != nil { +// continue +// } + +// var portObj []PortObj + +// ipAddress := jsonParsed.S("IPAddress").Data().(string) +// rawPorts := jsonParsed.S("Ports").Bytes() +// err = jsoniter.Unmarshal(rawPorts, &portObj) +// if err != nil || len(portObj) == 0 { +// continue +// } + +// var ports []string +// for _, p := range portObj { +// info := fmt.Sprintf("%v/%v/%v", p.PortID, p.Protocol, p.Service.Product) +// ports = append(ports, strings.Trim(info, "/")) +// } +// if len(ports) == 0 { +// continue +// } + +// obj := database.Dns{ +// DnsValue: ipAddress, +// DnsType: "A", +// Ports: strings.Join(ports, ","), +// ScanRefer: r.ScanObj.ID, +// TargetRefer: r.ScanObj.TargetRefer, +// } +// obj.UpdatePort() +// } +// } + +// // ImportJaelesVulnJson import new asset to DB +// func (r *Runner) ImportJaelesVulnJson(src string) { +// if !utils.FileExists(src) { +// utils.ErrorF("file not found: %v", src) +// return +// } +// content := utils.ReadingLines(src) +// baseDir := path.Dir(src) +// for _, line := range content { +// if strings.TrimSpace(line) == "" { +// continue +// } + +// jsonParsed, err := gabs.ParseJSON([]byte(line)) +// if err != nil { +// utils.ErrorF("Error parse JSON Data") +// continue +// } +// raw := jsonParsed.S("OutputFile").Data().(string) +// reportPath := raw +// if !utils.FileExists(reportPath) { +// reportPath = path.Join(baseDir, raw) +// if !utils.FileExists(reportPath) { +// reportPath = path.Join(path.Dir(baseDir), raw) +// } +// } +// vulnContent := utils.GetFileContent(reportPath) + +// // parse Data as JSON +// jsonParsed, err = gabs.ParseJSON([]byte(vulnContent)) +// if err != nil { +// utils.ErrorF("Error parse JSON Data") +// continue +// } + +// obj := database.Vulnerability{ +// URL: jsonParsed.S("URL").Data().(string), +// VulnRequest: jsonParsed.S("Req").Data().(string), +// VulnResponse: jsonParsed.S("Res").Data().(string), +// DetectionString: jsonParsed.S("DetectionString").Data().(string), +// VulnerabilityTitle: jsonParsed.S("SignName").Data().(string), +// SignatureID: jsonParsed.S("SignID").Data().(string), +// Confidence: jsonParsed.S("Confidence").Data().(string), +// Severity: jsonParsed.S("Risk").Data().(string), +// Source: "Jaeles", +// //VulnChecksum: utils.GenHash(vulnData), +// //PluginScan: SelectScanID(workspace, pluginName, scanID), +// //Target: SelectScanByWS(workspace), +// //Asset: SelectAssetByData(domain), +// ScanRefer: r.ScanObj.ID, +// TargetRefer: r.ScanObj.TargetRefer, +// } +// obj.Create() +// } +// } + +// // ImportNucleiVulnJson import new asset to DB +// func (r *Runner) ImportNucleiVulnJson(src string) { +// if !utils.FileExists(src) { +// utils.ErrorF("file not found: %v", src) +// return +// } +// content := utils.ReadingLines(src) +// for _, line := range content { +// if strings.TrimSpace(line) == "" { +// continue +// } +// jsonParsed, err := gabs.ParseJSON([]byte(line)) +// if err != nil { +// utils.ErrorF("Error parse JSON Data") +// continue +// } + +// obj := database.Vulnerability{ +// URL: jsonParsed.S("host").Data().(string), +// VulnRequest: utils.Base64Encode(jsonParsed.S("request").Data().(string)), +// VulnResponse: utils.Base64Encode(jsonParsed.S("response").Data().(string)), +// DetectionString: jsonParsed.S("matched").Data().(string), + +// SignatureID: jsonParsed.S("templateID").Data().(string), +// Confidence: "Tentative", +// VulnerabilityTitle: jsonParsed.S("info", "name").Data().(string), +// Severity: jsonParsed.S("info", "severity").Data().(string), +// Source: "Nuclei", +// TargetRefer: r.ScanObj.TargetRefer, + +// ScanRefer: r.ScanObj.ID, +// } +// obj.Create() +// } +// } + +// // ImportDirectoryJson import new asset to DB +// func (r *Runner) ImportDirectoryJson(src string) { +// if !utils.FileExists(src) { +// utils.ErrorF("file not found: %v", src) +// return +// } +// content := utils.ReadingLines(src) +// for _, line := range content { +// if strings.TrimSpace(line) == "" || !strings.Contains(line, "url") { +// continue +// } + +// jsonParsed, err := gabs.ParseJSON([]byte(line)) +// if err != nil { +// utils.ErrorF("Error parse JSON Data") +// continue +// } + +// // some parser here +// URL := jsonParsed.S("url").Data().(string) +// redirect, ok := jsonParsed.S("redirectlocation").Data().(string) +// if ok { +// redirect = "" +// } + +// obj := database.Directory{ +// URL: URL, +// Status: cast.ToInt(jsonParsed.S("status").Data()), +// ContentLength: cast.ToInt(jsonParsed.S("length").Data()), +// Words: cast.ToInt(jsonParsed.S("words").Data()), +// RedirectURL: redirect, +// ScanRefer: r.ScanObj.ID, +// TargetRefer: r.ScanObj.TargetRefer, +// } +// obj.Create() +// } +// } + +// +//// UpdateJSONDns update dns in db +//func UpdateJSONDns(src string, options libs.Options) { +// content := utils.ReadingLines(src) +// if len(content) == 0 { +// utils.ErrorF("File not found: %v", src) +// return +// } +// +// utils.DebugF("Update JSON DNS: %v", src) +// //wsObj := database.SelectScanByWS(options.Scan.ROptions["Workspace"]) +// target := options.Scan.ROptions["Workspace"] +// +// for _, line := range content { +// jsonParsed, err := gabs.ParseJSON([]byte(line)) +// if err != nil { +// continue +// } +// +// domain, ok := jsonParsed.S("host").Data().(string) +// if !ok { +// continue +// } +// +// // filtered some unrelated domain +// if !strings.HasSuffix(domain, fmt.Sprintf(".%s", target)) { +// if domain != target { +// continue +// } +// } +// +// data := "N/A" +// isError := jsonParsed.S("status_code").Data().(string) +// if isError == "NXDOMAIN" { +// //database.NewAssetWithDns(wsObj, domain, data) +// continue +// } +// var results []string +// a := jsonParsed.S("a") +// if a != nil { +// for _, record := range a.Children() { +// data := fmt.Sprintf("A/%s", cast.ToString(record.Data())) +// results = append(results, data) +// } +// } +// +// cname := jsonParsed.S("cname") +// if cname != nil { +// for _, record := range cname.Children() { +// data := fmt.Sprintf("CNAME/%s", cast.ToString(record.Data())) +// results = append(results, data) +// } +// } +// +// mx := jsonParsed.S("mx") +// if mx != nil { +// for _, record := range mx.Children() { +// data := fmt.Sprintf("MX/%s", cast.ToString(record.Data())) +// results = append(results, data) +// } +// } +// +// ns := jsonParsed.S("ns") +// if ns != nil { +// for _, record := range ns.Children() { +// data := fmt.Sprintf("NS/%s", cast.ToString(record.Data())) +// results = append(results, data) +// } +// } +// results = funk.UniqString(results) +// data = strings.Join(results, ";;") +// //database.NewAssetWithDns(wsObj, domain, data) +// } +//} diff --git a/core/import_mics.go b/core/import_mics.go new file mode 100644 index 0000000..24226d2 --- /dev/null +++ b/core/import_mics.go @@ -0,0 +1,238 @@ +package core + +// "github.com/j3ssie/osmedeus/database" + +// func (r *Runner) StartScanNoti() error { +// data, err := jsoniter.Marshal(&r.ScanObj) +// if err != nil { +// return fmt.Errorf("err marshal object: %v", err) +// } +// noti := database.Notification{ +// NotificationType: "start", +// NotificationSource: "scan", +// NewData: datatypes.JSON(data), +// ObjRefer: r.ScanObj.ID, +// ScanRefer: r.ScanObj.ID, +// TargetRefer: r.TargetObj.ID, +// } +// return noti.Create() +// } + +// func (r *Runner) ScanDoneNoti() error { +// data, err := jsoniter.Marshal(&r.ScanObj) +// if err != nil { +// return fmt.Errorf("err marshal object: %v", err) +// } +// noti := database.Notification{ +// NotificationType: "done", +// NotificationSource: "scan", +// NewData: datatypes.JSON(data), +// ObjRefer: r.ScanObj.ID, +// ScanRefer: r.ScanObj.ID, +// TargetRefer: r.TargetObj.ID, +// } +// return noti.Create() +// } + +// // ImportLinks import new link to DB +// func (r *Runner) ImportLinks(src string) { +// if !utils.FileExists(src) { +// utils.ErrorF("file not found: %v", src) +// return +// } + +// content := utils.ReadingLines(src) +// for _, line := range content { +// if strings.TrimSpace(line) == "" { +// continue +// } +// // {"input":"http://academy.live-test.redacted.io","source":"body","type":"url","output":"http://academy.live-test.redacted.io","status":307,"length":7} +// jsonParsed, err := gabs.ParseJSON([]byte(line)) +// if err != nil { +// utils.ErrorF("Error parse JSON Data") +// continue +// } + +// obj := database.Link{ +// LinkValue: jsonParsed.S("output").Data().(string), +// LinkSource: jsonParsed.S("source").Data().(string), +// URL: jsonParsed.S("input").Data().(string), +// LinkType: jsonParsed.S("type").Data().(string), +// ScanRefer: r.ScanObj.ID, +// TargetRefer: r.TargetObj.ID, +// } +// obj.Create() +// } +// } + +// func (r *Runner) ImportArchive(src string) { +// if !utils.FileExists(src) { +// utils.ErrorF("file not found: %v", src) +// return +// } + +// content := utils.ReadingLines(src) +// for _, line := range content { +// if strings.TrimSpace(line) == "" { +// continue +// } + +// // ignore root domain +// u, err := url.Parse(line) +// if err != nil || u.Path == "" || u.Path == "/" { +// continue +// } + +// obj := database.Archive{ +// ArchiveValue: line, +// ArchiveChecksum: utils.GenHash(line), +// ScanRefer: r.ScanObj.ID, +// TargetRefer: r.TargetObj.ID, +// } +// obj.Create() +// } +// } + +// func (r *Runner) ImportIPRange(src string) { +// if !utils.FileExists(src) { +// utils.ErrorF("file not found: %v", src) +// return +// } + +// content := utils.ReadingLines(src) +// for _, line := range content { +// if strings.TrimSpace(line) == "" { +// continue +// } +// jsonParsed, err := gabs.ParseJSON([]byte(line)) +// if err != nil { +// utils.ErrorF("Error parse JSON Data") +// continue +// } + +// utils.DebugF("Processing: %v", line) +// obj := database.IPRange{ +// ASNumber: cast.ToString(jsonParsed.S("Number").Data()), +// Country: cast.ToString(jsonParsed.S("CountryCode").Data()), +// Value: cast.ToString(jsonParsed.S("CIDR").Data()), +// Info: cast.ToString(jsonParsed.S("Description").Data()), +// Amount: cast.ToUint(jsonParsed.S("Count").Data()), +// ScanRefer: r.ScanObj.ID, +// TargetRefer: r.TargetObj.ID, +// } +// obj.Create() +// } +// } + +// func (r *Runner) ImportCert(src string) { +// if !utils.FileExists(src) { +// utils.ErrorF("file not found: %v", src) +// return +// } + +// content := utils.ReadingLines(src) +// for _, line := range content { +// if strings.TrimSpace(line) == "" { +// continue +// } +// jsonParsed, err := gabs.ParseJSON([]byte(line)) +// if err != nil { +// utils.ErrorF("Error parse JSON Data") +// continue +// } + +// domain := cast.ToString(jsonParsed.S("Domain").Data()) +// isWildCard := false +// if strings.Contains(domain, "*.") { +// isWildCard = true +// domain = strings.TrimLeft(domain, "*.") +// } + +// obj := database.CertInfo{ +// Domain: domain, +// CertInfo: cast.ToString(jsonParsed.S("CertInfo").Data()), +// OrgInfo: cast.ToString(jsonParsed.S("OrgInfo").Data()), +// IsWildcard: isWildCard, +// TargetRefer: r.TargetObj.ID, +// } +// obj.Create() +// } +// } + +// func (r *Runner) ImportCred(src string) { +// if !utils.FileExists(src) { +// utils.ErrorF("file not found: %v", src) +// return +// } + +// content := utils.ReadingLines(src) +// for _, line := range content { +// if strings.TrimSpace(line) == "" { +// continue +// } + +// jsonParsed, err := gabs.ParseJSON([]byte(line)) +// if err != nil { +// utils.ErrorF("Error parse JSON Data") +// continue +// } + +// email := cast.ToString(jsonParsed.S("email").Data()) +// credID := cast.ToString(jsonParsed.S("id").Data()) +// username := cast.ToString(jsonParsed.S("username").Data()) +// password := cast.ToString(jsonParsed.S("password").Data()) +// hashedPassword := cast.ToString(jsonParsed.S("hashed_password").Data()) +// name := cast.ToString(jsonParsed.S("name").Data()) +// ipAddress := cast.ToString(jsonParsed.S("ip_address").Data()) +// phone := cast.ToString(jsonParsed.S("phone").Data()) +// source := cast.ToString(jsonParsed.S("database_name").Data()) + +// obj := database.Credential{ +// CredID: credID, +// Email: email, +// Username: username, +// Password: password, +// HashedPassword: hashedPassword, +// Name: name, +// Phone: phone, +// IPAddress: ipAddress, +// Source: source, +// TargetRefer: r.TargetObj.ID, +// ScanRefer: r.ScanObj.ID, +// } +// obj.Create() +// } +// } + +// func (r *Runner) ImportCloudBrute(src string) { +// if !utils.FileExists(src) { +// utils.ErrorF("file not found: %v", src) +// return +// } + +// content := utils.ReadingLines(src) +// for _, line := range content { +// if strings.TrimSpace(line) == "" { +// continue +// } + +// if !strings.Contains(line, " - ") { +// continue +// } + +// cloudDomain := strings.Split(line, " - ")[1] +// status := strings.Split(line, " - ")[0] +// if strings.Contains(status, ": ") { +// status = strings.Split(strings.Split(line, " - ")[0], ": ")[1] +// } + +// obj := database.CloudBrute{ +// Status: status, +// CloudDomain: cloudDomain, +// RawData: line, +// ScanRefer: r.ScanObj.ID, +// TargetRefer: r.TargetObj.ID, +// } +// obj.Create() +// } +// } diff --git a/core/markdown.go b/core/markdown.go new file mode 100644 index 0000000..e32bde0 --- /dev/null +++ b/core/markdown.go @@ -0,0 +1,256 @@ +package core + +import ( + "path" + + "fmt" + "os" + "regexp" + "strings" + + "github.com/fatih/color" + "github.com/j3ssie/osmedeus/libs" + "github.com/j3ssie/osmedeus/utils" + "github.com/spf13/cast" + + "github.com/gomarkdown/markdown" + "github.com/gomarkdown/markdown/html" + "github.com/gomarkdown/markdown/parser" +) + +func (r *Runner) GenMarkdownReport(markdownFile string, outputHTML string) { + utils.DebugF("Reading markdown report from: %v", markdownFile) + + // get the markdown template content + mdContent := utils.GetFileContent(markdownFile) + mdContent = ResolveData(mdContent, r.Target) + + // replace all the tag + mdContent = r.ResolveScanInfoTag(mdContent) + // utils.DebugF("ResolveScanInfoTag:\n%v", mdContent) + + // replace all the tag + mdContent = r.ResolveReportsTag(mdContent) + + // replace all the tag + mdContent = r.ResolveContentTag(mdContent) + // fmt.Println("mdContent", mdContent) + + // generating the markdown file first + outputMD := strings.Replace(outputHTML, ".html", ".md", -1) + utils.WriteToFile(outputMD, mdContent) + + utils.InforF("Generate markdown report: %v", outputMD) + utils.InforF("Generate HTML report: %v", outputHTML) + // finally convert to HTML + MarkDownToHTML(r.Opt, r.Input, outputMD, outputHTML) +} + +func (r *Runner) ResolveScanInfoTag(rawMarkdown string) string { + re := regexp.MustCompile(``) + match := re.FindString(rawMarkdown) + if len(match) > 1 { + utils.DebugF("Replace scanInfo tag: %v", match) + scanInfo := fmt.Sprintf(` +| | | +|----------------|-------------| +| Target | **:target** | +| Running Time | **:runningTime** | +| Workflow | **:workflow** | +| Status | **:status** | +| Statistics | :statistics | +`) + + status := "done" + if r.ScanObj.IsRunning { + status = "running" + } + + statistics := fmt.Sprintf("`assets/%v`, `dns/%v`, `vulnerability/%v`, ", r.TargetObj.TotalAssets, r.TargetObj.TotalDns, r.TargetObj.TotalVulnerability) + replacements := map[string]string{ + ":target": r.ScanObj.InputName, + ":runningTime": cast.ToString(int(r.RunningTime)/3600) + " hours", + ":workflow": r.ScanObj.TaskName, + ":statistics": statistics, + ":status": status, + } + // generate the statistics info + for oldStr, newStr := range replacements { + scanInfo = strings.ReplaceAll(scanInfo, oldStr, newStr) + } + + return strings.Replace(rawMarkdown, match, scanInfo, -1) + } + utils.DebugF("No scanInfo tag found") + return rawMarkdown +} + +func (r *Runner) ResolveContentTag(rawData string) string { + finalMarkdown := rawData + // finding all the content tags and replace it with the content + re := regexp.MustCompile(`]*>`) + matchs := re.FindAllString(rawData, -1) + for _, contentTag := range matchs { + utils.DebugF("Replace content tag: %v", color.GreenString(contentTag)) + content := r.ResolveContentSrc(contentTag) + finalMarkdown = strings.Replace(finalMarkdown, contentTag, content, -1) + } + + return finalMarkdown +} + +func (r *Runner) ResolveContentSrc(tag string) string { + re := regexp.MustCompile(`src=\"(\S+)\"`) + match := re.FindStringSubmatch(tag) + if len(match) > 1 { + fileContent := utils.GetFileContent(match[1]) + utils.DebugF("Replace content src: %v", color.GreenString(match[1])) + + if strings.Contains(tag, "expand=true") { + return "```\n" + fileContent + "```" + } + + if strings.Contains(tag, "shorten=true") || len(fileContent) > r.Opt.MDCodeBlockLimit { + return extendTag(fileContent) + } + + return "```\n" + fileContent + "```" + } + return "" +} + +func extendTag(str string) string { + data := "
\nClick to Expand\n\n" + "
\n" + str + "\n
" + "\n
" + return data +} + +func (r *Runner) ResolveReportsTag(rawMarkDown string) string { + // rawData := utils.GetFileContent(markdownFile) + finalMarkdown := rawMarkDown + // finding all the reports tags + re := regexp.MustCompile(``) + matchs := re.FindAllString(rawMarkDown, -1) + if len(matchs) == 0 { + utils.DebugF("No reports tag found") + return rawMarkDown + } + + for _, reportTag := range matchs { + utils.DebugF("Replace content tag: %v", reportTag) + mdContent := "" + + for _, report := range r.TargetObj.Reports { + // add the link if report file is HTML file + if report.ReportType == "html" { + mdContent += fmt.Sprintf("### %s -- [%s](%s) \n\n", report.Module, report.ReportName, report.ReportPath) + mdContent += "\n***\n" + continue + } + // add the full content if report file is a text file + mdContent += fmt.Sprintf("### %s -- *%s* \n\n", report.Module, report.ReportName) + + fileContent := utils.GetFileContent(report.ReportPath) + if len(fileContent) > r.Opt.MDCodeBlockLimit { + mdContent += extendTag(fileContent) + } else { + mdContent += "```\n" + mdContent += fileContent + mdContent += "\n```\n" + } + mdContent += "\n***\n\n" + } + + finalMarkdown = strings.Replace(finalMarkdown, reportTag, mdContent, -1) + } + + return finalMarkdown +} + +func MarkDownToHTML(options libs.Options, target string, markdownFile string, outputFile string) error { + css := path.Join(options.Env.DataFolder, "markdown/style.css") + + var input []byte + var err error + + if input, err = os.ReadFile(markdownFile); err != nil { + utils.ErrorF("Error reading %s: %v", markdownFile, err) + return err + } + + // set up options + var extensions = parser.NoIntraEmphasis | + parser.Tables | + parser.FencedCode | + parser.Autolink | + parser.Strikethrough | + parser.SpaceHeadings + + var renderer markdown.Renderer + // render the data into HTML + var htmlFlags html.Flags + + htmlFlags |= html.Smartypants + htmlFlags |= html.UseXHTML + htmlFlags |= html.CompletePage + htmlFlags |= html.SmartypantsLatexDashes + htmlFlags |= html.SmartypantsFractions + + params := html.RendererOptions{ + Flags: htmlFlags, + CSS: css, + } + renderer = html.NewRenderer(params) + + // parse and render + var output []byte + parser := parser.NewWithExtensions(extensions) + // @NOTE: beware of XSS as I assume you will trust the markdown content you generate + output = markdown.ToHTML(input, parser, renderer) + // html := bluemonday.UGCPolicy().SanitizeBytes(output) // skip the sanitization as we prefer more beautiful output + + cssContent, err := os.ReadFile(css) + if err != nil { + return err + } + + finalHTML := fmt.Sprintf(` + + + + + + + + Osmedeus Executive Summary - %s + + + + + + + + + + + + +%s + + + `, target, string(cssContent), string(output)) + + // output the result + var out *os.File + if out, err = os.Create(outputFile); err != nil { + utils.ErrorF("Error creating %s: %v", outputFile, err) + return err + } + defer out.Close() + + if _, err = out.WriteString(finalHTML); err != nil { + utils.ErrorF("Error writing output: %v", err) + return err + } + + return nil +} diff --git a/core/mode_test.go b/core/mode_test.go new file mode 100644 index 0000000..4d4cbc0 --- /dev/null +++ b/core/mode_test.go @@ -0,0 +1,24 @@ +package core + +import ( + "fmt" + "testing" + + "github.com/j3ssie/osmedeus/libs" +) + +func TestListMode(t *testing.T) { + var options libs.Options + options.Env.WorkFlowsFolder = "~/go/src/github.com/j3ssie/osmedeus/workflow/" + result := ListFlow(options) + fmt.Println(result) + if len(result) == 0 { + t.Errorf("Error ListMode") + } + + selectedMode := SelectFlow("general", options) + fmt.Println(selectedMode) + if len(selectedMode) == 0 { + t.Errorf("Error selectedMode") + } +} diff --git a/core/module.go b/core/module.go new file mode 100644 index 0000000..5b061b9 --- /dev/null +++ b/core/module.go @@ -0,0 +1,370 @@ +package core + +import ( + "context" + "fmt" + "path" + "strings" + "sync" + "time" + + "github.com/spf13/cast" + + "github.com/fatih/color" + "github.com/j3ssie/osmedeus/execution" + "github.com/j3ssie/osmedeus/libs" + "github.com/j3ssie/osmedeus/utils" + "github.com/panjf2000/ants" +) + +// RunModule run the module +func (r *Runner) RunModule(module libs.Module) { + // get reports path + module = ResolveReports(module, r.Params) + + // check if resume enable or not + if (r.Opt.Resume || module.Resume) && !module.Forced { + if CheckResume(module) { + utils.BlockF(module.Name, "Resume detected") + return + } + } + + r.CurrentModule = module.Name + timeStart := time.Now() + utils.BlockF("Module-Started", fmt.Sprintf("%v - %v", module.Name, module.Desc)) + + // create report record first because I don't want to wait for them to show up in UI until the module done + r.DBNewReports(module) + + // pre-run + if len(module.PreRun) > 0 && r.Opt.NoPreRun == false { + utils.InforF("Running prepare scripts for module %v", color.CyanString(module.Name)) + r.RunScripts(module.PreRun) + } + + // main part + // utils.BlockF(module.Name, "Begin executing primary tasks") + err := r.RunSteps(module.Steps) + if err != nil { + utils.BadBlockF(module.Name, fmt.Sprintf("got exit call")) + } + + // post-run + if len(module.PostRun) > 0 && r.Opt.NoPostRun == false { + utils.InforF("Running conclude scripts for module %v", color.CyanString(module.Name)) + r.RunScripts(module.PostRun) + } + + // print the reports file + utils.PrintLine() + printReports(module) + + // estimate time + elapsedTime := time.Since(timeStart).Seconds() + utils.BlockF("Module-Ended", fmt.Sprintf("Elapsed Time for the module %v in %v", color.HiCyanString(module.Name), color.HiMagentaString("%vs", elapsedTime))) + r.RunningTime += cast.ToInt(elapsedTime) + utils.PrintLine() + r.DBUpdateScan() +} + +// RunScripts run list of scripts +func (r *Runner) RunScripts(scripts []string) string { + if r.Opt.Timeout != "" { + timeout := utils.CalcTimeout(r.Opt.Timeout) + utils.DebugF("Run scripts with %v seconds timeout", timeout) + r.RunScriptsWithTimeOut(r.Opt.Timeout, scripts) + return "" + } + + for _, script := range scripts { + outScript := r.RunScript(script) + if strings.Contains(outScript, "exit") { + return outScript + } + } + return "" +} + +// RunScriptsWithTimeOut run list of scripts with timeout +func (r *Runner) RunScriptsWithTimeOut(timeoutRaw string, scripts []string) string { + timeout := utils.CalcTimeout(timeoutRaw) + utils.DebugF("Run scripts with %v seconds timeout", timeout) + + c := context.Background() + deadline := time.Now().Add(time.Duration(timeout) * time.Second) + c, cancel := context.WithDeadline(c, deadline) + defer cancel() + + go func() { + for _, script := range scripts { + outScript := r.RunScript(script) + if strings.Contains(outScript, "exit") { + return + } + } + cancel() + }() + + select { + case <-c.Done(): + utils.DebugF("Scripts done") + return "" + case <-time.After(time.Duration(timeout) * time.Second): + utils.BadBlockF("timeout", fmt.Sprintf("Scripts got timeout after %v", color.HiMagentaString(timeoutRaw))) + } + return "" +} + +// RunScript really run a script +func (r *Runner) RunScript(script string) string { + return r.ExecScript(script) +} + +// RunSteps run list of steps +func (r *Runner) RunSteps(steps []libs.Step) error { + var stepOut string + for _, step := range steps { + r.DoneStep += 1 + + if step.Timeout != "" { + // timeout should be: 30, 30m, 1h + timeout := utils.CalcTimeout(step.Timeout) + if timeout != 0 { + stepOut, _ = r.RunStepWithTimeout(timeout, step) + if strings.Contains(stepOut, "exit") { + return fmt.Errorf("got exit call") + } + continue + } + } + + stepOut, _ = r.RunStep(step) + if strings.Contains(stepOut, "exit") { + return fmt.Errorf("got an exit call") + } + } + return nil +} + +// RunStepWithTimeout run step with timeout +func (r *Runner) RunStepWithTimeout(timeout int, step libs.Step) (out string, err error) { + utils.DebugF("Run step with %v seconds timeout", timeout) + prefix := fmt.Sprintf("timeout -k 1m %vs ", timeout) + + // prepare the os command with prefix timeout first + var preFixCommands []string + for _, command := range step.Commands { + preFixCommand := command + if !strings.Contains(command, "timeout") { + preFixCommand = prefix + command + } + preFixCommands = append(preFixCommands, preFixCommand) + } + step.Commands = preFixCommands + + // override global timeout + r.Opt.Timeout = step.Timeout + return r.RunStep(step) +} + +func (r *Runner) RunStep(step libs.Step) (string, error) { + var output string + if step.Label != "" { + utils.BlockF("Step", fmt.Sprintf("Initiating Step %v", color.HiGreenString(step.Label))) + } + + // checking required file + err := r.CheckRequired(step.Required) + if err != nil { + return output, fmt.Errorf("missing requirements") + } + + // check conditions and run reverse step + err = r.CheckCondition(step.Conditions) + if err != nil { + if len(step.RCommands) == 0 && len(step.RScripts) == 0 { + return output, fmt.Errorf("conditions not met") + } + + // run reverse commands + utils.InforF("Condition false, run the reverse commands") + if len(step.RCommands) > 0 { + r.RunCommands(step.RCommands, step.Std) + } + // run reverse scripts + if len(step.RScripts) > 0 { + output = r.RunScripts(step.RScripts) + if strings.Contains(output, "exit") { + return output, nil + } + } + return output, nil + } + + // run the step in loop mode + if step.Source != "" { + return r.RunStepWithSource(step) + } + // + + if len(step.Commands) > 0 { + r.RunCommands(step.Commands, step.Std) + } + if len(step.Scripts) > 0 { + output = r.RunScripts(step.Scripts) + if strings.Contains(output, "exit") { + return output, nil + } + } + + // run ose here + if len(step.Ose) > 0 { + for _, ose := range step.Ose { + r.RunOse(ose) + } + } + + // post scripts + if len(step.PConditions) > 0 || len(step.PScripts) > 0 { + err := r.CheckCondition(step.PConditions) + if err == nil { + if len(step.PScripts) > 0 { + r.RunScripts(step.PScripts) + } + } + } + return output, nil + +} + +// RunStepWithSource really run a step +func (r *Runner) RunStepWithSource(step libs.Step) (out string, err error) { + ////// Start to run step but in loop mode + utils.DebugF("Running the step using the source file: %v", step.Source) + data := utils.ReadingLines(step.Source) + if len(data) <= 0 { + return out, fmt.Errorf("missing source") + } + if step.Threads != "" { + step.Parallel = cast.ToInt(step.Threads) + } + if step.Parallel == 0 { + step.Parallel = 1 + } + + // prepare the data first + var newGeneratedSteps []libs.Step + for index, line := range data { + customParams := make(map[string]string) + customParams["line"] = line + customParams["line_id"] = fmt.Sprintf("%v-%v", path.Base(line), index) + customParams["_id_"] = fmt.Sprintf("%v", index) + customParams["_line_"] = execution.StripName(line) + + // make completely new Step + localStep := libs.Step{} + + for _, cmd := range step.Commands { + localStep.Commands = append(localStep.Commands, AltResolveVariable(cmd, customParams)) + } + for _, cmd := range step.RCommands { + localStep.RCommands = append(localStep.RCommands, AltResolveVariable(cmd, customParams)) + } + + if len(step.Ose) > 0 { + for _, ose := range step.Ose { + localStep.Ose = append(localStep.Ose, AltResolveVariable(ose, customParams)) + } + } + + for _, script := range step.RScripts { + localStep.RScripts = append(localStep.RScripts, AltResolveVariable(script, customParams)) + } + + for _, script := range step.Scripts { + localStep.Scripts = append(localStep.Scripts, AltResolveVariable(script, customParams)) + } + + for _, script := range step.PConditions { + localStep.PConditions = append(localStep.PConditions, AltResolveVariable(script, customParams)) + } + for _, script := range step.PScripts { + localStep.PScripts = append(localStep.PScripts, AltResolveVariable(script, customParams)) + } + + newGeneratedSteps = append(newGeneratedSteps, localStep) + } + + // skip concurrency part + if step.Parallel == 1 { + for _, newGeneratedStep := range newGeneratedSteps { + out, err = r.RunStep(newGeneratedStep) + if err != nil { + continue + } + } + } else { + ///////////// + // run multiple steps in concurrency mode + + utils.DebugF("Running the step in parallel: %v", step.Parallel) + var wg sync.WaitGroup + p, _ := ants.NewPoolWithFunc(step.Parallel, func(i interface{}) { + r.startStepJob(i) + wg.Done() + }, ants.WithPreAlloc(true)) + defer p.Release() + + for _, newGeneratedStep := range newGeneratedSteps { + wg.Add(1) + err = p.Invoke(newGeneratedStep) + if err != nil { + utils.ErrorF("Error in parallel: %v", err) + } + } + + wg.Wait() + } + return out, nil +} + +func (r *Runner) startStepJob(j interface{}) { + localStep := j.(libs.Step) + + err := r.CheckCondition(localStep.Conditions) + + if err != nil { + // run reverse commands + if len(localStep.RCommands) > 0 { + r.RunCommands(localStep.RCommands, localStep.Std) + } + if len(localStep.RScripts) > 0 { + r.RunScripts(localStep.RScripts) + } + } else { + if len(localStep.Commands) > 0 { + r.RunCommands(localStep.Commands, localStep.Std) + } + } + + if len(localStep.Ose) > 0 { + for _, ose := range localStep.Ose { + r.RunOse(ose) + } + } + + if len(localStep.Scripts) > 0 { + r.RunScripts(localStep.Scripts) + } + + // post scripts + if len(localStep.PConditions) > 0 || len(localStep.PScripts) > 0 { + err := r.CheckCondition(localStep.PConditions) + if err == nil { + if len(localStep.PScripts) > 0 { + r.RunScripts(localStep.PScripts) + } + } + } +} diff --git a/core/parse.go b/core/parse.go new file mode 100644 index 0000000..47125a3 --- /dev/null +++ b/core/parse.go @@ -0,0 +1,322 @@ +package core + +import ( + "bytes" + "fmt" + "net/url" + "os" + "path" + "path/filepath" + "strings" + "text/template" + "time" + + "github.com/Jeffail/gabs/v2" + "github.com/fatih/color" + "github.com/flosch/pongo2/v6" + "github.com/spf13/cast" + "golang.org/x/net/publicsuffix" + + "github.com/Shopify/yaml" + "github.com/j3ssie/osmedeus/libs" + "github.com/j3ssie/osmedeus/utils" +) + +// ResolveData resolve template from signature file +func ResolveData(format string, data map[string]string) string { + // for backward compatibility because new template using `{{variable}}` instead of `{{.variable}}` + if strings.Contains(format, "{{.") { + return OldResolveData(format, data) + } + + variable := make(map[string]interface{}) + for k, v := range data { + variable[k] = v + } + if tpl, err := pongo2.FromString(format); err == nil { + out, ok := tpl.Execute(variable) + if ok == nil { + return out + } + utils.ErrorF("Error when resolve template: %v", ok) + } + return format +} + +// OldResolveData resolve template from signature file +func OldResolveData(format string, data map[string]string) string { + t := template.Must(template.New("").Parse(format)) + + buf := &bytes.Buffer{} + err := t.Execute(buf, data) + if err != nil { + utils.ErrorF("Error render: %v -- %v", format, err) + return format + } + return buf.String() +} + +// ResolveSlice resolve template from signature file +func ResolveSlice(slice []string, data map[string]string) (resolveSlice []string) { + for _, s := range slice { + resolveSlice = append(resolveSlice, ResolveData(s, data)) + } + return resolveSlice +} + +// AltResolveVariable just like ResolveVariable but looking for [[.var]] +func AltResolveVariable(format string, data map[string]string) string { + t := template.Must(template.New("").Delims("[[", "]]").Parse(format)) + buf := &bytes.Buffer{} + err := t.Execute(buf, data) + if err != nil { + return format + } + return buf.String() +} + +// ParseFlow parse mode file +func ParseFlow(flowFile string) (libs.Flow, error) { + utils.DebugF("Parsing workflow at: %v", color.HiGreenString(flowFile)) + var flow libs.Flow + yamlFile, err := os.ReadFile(flowFile) + if err != nil { + utils.ErrorF("YAML parsing err: %v -- #%v ", flowFile, err) + return flow, err + } + err = yaml.Unmarshal(yamlFile, &flow) + if err != nil { + utils.ErrorF("Error unmarshal: %v -- %v", flowFile, err) + return flow, err + } + + if flow.Usage != "" && strings.Contains(flow.Usage, "{{.this_file}}") { + flow.Usage = strings.ReplaceAll(flow.Usage, "{{.this_file}}", flowFile) + } + return flow, nil +} + +// ParseModules parse module file +func ParseModules(moduleFile string) (libs.Module, error) { + utils.DebugF("Parsing module at: %v", color.HiCyanString(moduleFile)) + + var module libs.Module + + yamlFile, err := os.ReadFile(moduleFile) + if err != nil { + utils.ErrorF("YAML parsing err: %v -- #%v ", moduleFile, err) + return module, err + } + err = yaml.Unmarshal(yamlFile, &module) + if err != nil { + utils.ErrorF("Error unmarshal: %v -- %v", moduleFile, err) + return module, err + } + module.ModulePath = moduleFile + if module.Usage != "" && strings.Contains(module.Usage, "{{this_file}}") { + module.Usage = strings.ReplaceAll(module.Usage, "{{this_file}}", moduleFile) + } + return module, err +} + +// ParseInputFormat format input +func ParseInputFormat(raw string, options libs.Options) map[string]string { + target := make(map[string]string) + target["RawFormat"] = raw + + jsonParsed, err := gabs.ParseJSON([]byte(raw)) + if err != nil { + return target + } + + // parse Target first if found one + rawURL, ok := jsonParsed.ChildrenMap()["Target"] + if !ok { + panic("missing Target in special input") + } + + target = ParseInput(cast.ToString(rawURL.Data()), options) + + // override the whole thing + for k, v := range jsonParsed.ChildrenMap() { + target[k] = fmt.Sprintf("%v", v.Data()) + } + + return target +} + +// ParseInput parse input for routine +func ParseInput(raw string, options libs.Options) map[string]string { + ROptions := ParseTarget(raw) + if options.EnableFormatInput { + // avoid the loophole + options.EnableFormatInput = false + ROptions = ParseInputFormat(raw, options) + return ROptions + } + // some data stuff + dir, err := os.Getwd() + if err == nil { + ROptions["CWD"] = dir + } + + // default threads variables + ROptions["Threads"] = cast.ToString(options.Threads) + ROptions["threads"] = cast.ToString(options.Threads) + ROptions["thread"] = cast.ToString(options.Threads) + ROptions["baseThreads"] = cast.ToString(options.Threads) + + ROptions["Version"] = libs.VERSION + ROptions["Bucket"] = options.Cdn.Bucket + + ROptions["Today"] = time.Now().Format("2006-01-02") + ROptions["CurrentDay"] = time.Now().Format("2006-01-02T15:04:05") + ROptions["Date"] = time.Now().Format("2006-01-02T15:04:05") + ROptions["TimeStamp"] = utils.GetTS() + ROptions["TS"] = time.Now().Format("2006-01-02") + "T" + utils.GetTS() + + /* --- start to load default Env --- */ + // ~/osmedeus-base + ROptions["BaseFolder"] = utils.NormalizePath(strings.TrimLeft(options.Env.BaseFolder, "/")) + ROptions["Plugins"] = options.Env.BinariesFolder + ROptions["Binaries"] = options.Env.BinariesFolder + + ROptions["Backup"] = options.Env.BackupFolder + ROptions["Data"] = options.Env.DataFolder + ROptions["Workflow"] = options.Env.WorkFlowsFolder + ROptions["Scripts"] = options.Env.WorkFlowsFolder + ROptions["Cloud"] = options.Env.CloudConfigFolder + + ROptions["Workspaces"] = options.Env.WorkspacesFolder + if options.Scan.BaseWorkspace != "" { + ROptions["Workspaces"] = options.Scan.BaseWorkspace + } + ROptions["Storages"] = options.Env.StoragesFolder + /* --- end of load default Env --- */ + + ROptions["Workspace"] = utils.CleanPath(raw) + if options.Scan.CustomWorkspace != "" { + ROptions["Workspace"] = utils.CleanPath(options.Scan.CustomWorkspace) + } + ROptions["Output"] = path.Join(ROptions["Workspaces"], ROptions["Workspace"]) + + // params in workflow file + if len(options.Flow.Params) > 0 { + for _, param := range options.Flow.Params { + for k, v := range param { + v = ResolveData(v, ROptions) + if strings.HasPrefix(v, "~/") { + v = utils.NormalizePath(v) + } + ROptions[k] = v + } + } + } + + return ROptions +} + +// ParseParams parse more params from cli +func ParseParams(rawParams []string) map[string]string { + params := make(map[string]string) + for _, item := range rawParams { + if strings.Contains(item, "=") { + data := strings.Split(item, "=") + params[data[0]] = strings.Replace(item, data[0]+"=", "", -1) + } + } + return params +} + +// ParseTarget parsing target and some variable for template +func ParseTarget(raw string) map[string]string { + target := make(map[string]string) + if raw == "" { + return target + } + target["Target"] = raw + u, err := url.Parse(raw) + + // something wrong so parsing it again + if err != nil || u.Scheme == "" || strings.Contains(u.Scheme, ".") { + raw = fmt.Sprintf("https://%v", raw) + u, err = url.Parse(raw) + if err != nil { + return target + } + // fmt.Println("parse again") + } + var hostname string + var query string + port := u.Port() + // var domain string + domain := u.Hostname() + + query = u.RawQuery + if u.Port() == "" { + if strings.Contains(u.Scheme, "https") { + port = "443" + } else { + port = "80" + } + + hostname = u.Hostname() + } else { + // ignore common port in Host + if u.Port() == "443" || u.Port() == "80" { + hostname = u.Hostname() + } else { + hostname = u.Hostname() + ":" + u.Port() + } + } + + target["Scheme"] = u.Scheme + target["Path"] = u.Path + target["Domain"] = domain + + target["Org"] = domain + suffix, ok := publicsuffix.PublicSuffix(domain) + if ok { + target["Org"] = strings.Replace(domain, fmt.Sprintf(".%s", suffix), "", -1) + } else { + if strings.Contains(domain, ".") { + parts := strings.Split(domain, ".") + if len(parts) == 2 { + target["Org"] = parts[0] + } else { + target["Org"] = parts[len(parts)-2] + } + } + } + + target["Host"] = hostname + target["Port"] = port + target["RawQuery"] = query + + if (target["RawQuery"] != "") && (port == "80" || port == "443") { + target["URL"] = fmt.Sprintf("%v://%v%v?%v", target["Scheme"], target["Host"], target["Path"], target["RawQuery"]) + } else if port != "80" && port != "443" { + target["URL"] = fmt.Sprintf("%v://%v:%v%v?%v", target["Scheme"], target["Domain"], target["Port"], target["Path"], target["RawQuery"]) + } else { + target["URL"] = fmt.Sprintf("%v://%v%v", target["Scheme"], target["Host"], target["Path"]) + } + + uu, _ := url.Parse(raw) + target["BaseURL"] = fmt.Sprintf("%v://%v", uu.Scheme, uu.Host) + target["Extension"] = filepath.Ext(target["BaseURL"]) + + return target +} + +func IsRootDomain(raw string) bool { + suffix, ok := publicsuffix.PublicSuffix(raw) + if ok { + return false + } + + input := strings.ReplaceAll(raw, fmt.Sprintf(".%s", suffix), "") + if strings.Count(input, ".") == 1 && strings.Count(input, "/") == 0 { + return true + } + return false +} diff --git a/core/parse_test.go b/core/parse_test.go new file mode 100644 index 0000000..bd392dc --- /dev/null +++ b/core/parse_test.go @@ -0,0 +1,71 @@ +package core + +import ( + "fmt" + "github.com/flosch/pongo2/v6" + "runtime" + "testing" +) + +func TestParseTarget(t *testing.T) { + fmt.Println("----> INPUT:", "http://example.com") + result := ParseTarget("http://exmaple.com") + fmt.Println(result) + if len(result) == 0 { + t.Errorf("Error RunMasscan") + } + // case 2 + fmt.Println("----> INPUT:", "example.com") + result = ParseTarget("exmaple.com") + fmt.Println(result) + if len(result) == 0 { + t.Errorf("Error RunMasscan") + } + + // case 2 + fmt.Println("----> INPUT:", "http://exmaple.com/123?q=1") + result = ParseTarget("http://exmaple.com/123?q=1") + fmt.Println(result) + if len(result) == 0 { + t.Errorf("Error RunMasscan") + } + + // case 2 + fmt.Println("----> INPUT:", "1.2.3.4") + result = ParseTarget("1.2.3.4") + fmt.Println(result) + if len(result) == 0 { + t.Errorf("Error RunMasscan") + } + + // case 2 + fmt.Println("----> INPUT:", "1.2.3.4/24") + result = ParseTarget("1.2.3.4/24") + fmt.Println(result) + if len(result) == 0 { + t.Errorf("Error RunMasscan") + } +} + +func TestRenderTemplate(t *testing.T) { + formatString := "Hello {{name}}! \n" + formatString += "--> Express: {{ 2 * cpu }} \n" + formatString += "--> Bool: {{ Skip }} \n" + target := map[string]any{ + "name": "World", + "num": "2", + "cpu": runtime.NumCPU(), + "Skip": true, + } + fmt.Println(target) + + if tpl, err := pongo2.FromString(formatString); err == nil { + // Now you can render the template with the given + // pongo2.Context how often you want to. + out, err := tpl.Execute(target) + if err != nil { + panic(err) + } + fmt.Println(out) // Output: Hello Florian! + } +} diff --git a/core/queue.go b/core/queue.go new file mode 100644 index 0000000..52c8934 --- /dev/null +++ b/core/queue.go @@ -0,0 +1,147 @@ +package core + +import ( + "fmt" + "github.com/fatih/color" + "github.com/fsnotify/fsnotify" + "github.com/j3ssie/osmedeus/libs" + "github.com/j3ssie/osmedeus/utils" + jsoniter "github.com/json-iterator/go" + "github.com/panjf2000/ants" + "github.com/thoas/go-funk" + "os" + "strings" + "sync" +) + +func QueueWatcher(options libs.Options) { + queueFile := options.Queue.QueueFile + + var wg sync.WaitGroup + p, _ := ants.NewPoolWithFunc(options.Concurrency, func(i interface{}) { + RunTheScan(i.(string), options) + wg.Done() + }, ants.WithPreAlloc(true)) + defer p.Release() + + data := utils.ReadingFileUnique(queueFile) + + /* Start the watcher */ + + // Create new watcher. + watcher, err := fsnotify.NewWatcher() + if err != nil { + panic(err) + } + defer watcher.Close() + utils.InforF("Starting to watch the queue file: %v", color.HiMagentaString(queueFile)) + + // Start listening for events. + go func() { + for { + select { + case event, ok := <-watcher.Events: + if !ok { + return + } + if event.Op.String() == "WRITE" { + utils.DebugF(color.HiMagentaString("modified file: %v -- %v", event.Name, event.Op)) + target := GetNewLine(queueFile) + wg.Add(1) + _ = p.Invoke(strings.TrimSpace(target)) + } + + if event.Op.String() == "REMOVE" { + utils.ErrorF("Queue file removed, exiting ...") + os.Exit(-1) + } + case err, ok := <-watcher.Errors: + if !ok { + return + } + utils.ErrorF("error: %v", err) + } + } + }() + + err = watcher.Add(queueFile) + if err != nil { + panic(err) + } + + // Just to trigger the file events + utils.WriteToFile(queueFile, strings.Join(append(data, "\n"), "\n")) + + // Block main goroutine forever. + <-make(chan struct{}) +} + +func RunTheScan(target string, options libs.Options) error { + if strings.TrimSpace(target) == "" { + return fmt.Errorf("target is empty") + } + utils.InforF("Picking the target from the queue: %v", color.CyanString(target)) + + var inputFormat libs.InputFormat + if ok := jsoniter.UnmarshalFromString(target, &inputFormat); ok == nil { + utils.DebugF("Parsing the input in JSON format: %v", color.CyanString(target)) + cmd := CommandBuilder(inputFormat) + utils.InforF("Running the command: %v", color.CyanString(cmd)) + utils.RunOSCommand(cmd) + return nil + } + + runner, err := InitRunner(target, options) + if err != nil { + utils.ErrorF("Error init runner with: %s", target) + return err + } + runner.Start() + + return nil +} + +func GetNewLine(queueFile string) string { + data := utils.ReadingLines(queueFile) + if len(data) == 0 { + return "" + } + + target := data[0] + data = funk.DropString(data, 1) + utils.DebugF("Getting the target from the queue file: %v -- %v", queueFile, color.CyanString(target)) + utils.WriteToFile(queueFile, strings.Join(data, "\n")) + return target +} + +func CommandBuilder(inputFormat libs.InputFormat) (command string) { + if inputFormat.Command == "" { + inputFormat.Command = fmt.Sprintf("%v scan -t %v", libs.BINARY, inputFormat.Input) + if inputFormat.InputAsFile { + inputFormat.Command = fmt.Sprintf("%v scan -T %v", libs.BINARY, inputFormat.Input) + } + + if inputFormat.Flow == "" { + inputFormat.Command += " -f " + inputFormat.Flow + } + + // append the modules + if len(inputFormat.Modules) > 0 { + for _, item := range inputFormat.Modules { + inputFormat.Command += " -m " + item + } + } + + if len(inputFormat.Params) > 0 { + for _, item := range inputFormat.Params { + inputFormat.Command += " -p " + item + } + } + + inputFormat.Command += " " + inputFormat.Extra + } + + // formatting the command if there is any input in it + inputFormat.Command = strings.ReplaceAll(inputFormat.Command, "{{.input}}", inputFormat.Input) + return command +} diff --git a/core/reference.go b/core/reference.go new file mode 100644 index 0000000..9d8a7da --- /dev/null +++ b/core/reference.go @@ -0,0 +1,104 @@ +package core + +/* File to store all the script for better reference */ + +const ( + Cleaning = "Cleaning" + CleanAmass = "CleanAmass" + CleanRustScan = "CleanRustScan" + CleanGoBuster = "CleanGoBuster" + CleanMassdns = "CleanMassdns" + CleanSWebanalyze = "CleanSWebanalyze" + CleanJSONDnsx = "CleanJSONDnsx" + CleanWebanalyze = "CleanWebanalyze" + CleanArjun = "CleanArjun" + GenNucleiReport = "GenNucleiReport" + CleanJSONHttpx = "CleanJSONHttpx" + CleanFFUFJson = "CleanFFUFJson" +) + +const ( + // noti for slack + StartNoti = "StartNoti" + DoneNoti = "DoneNoti" + ReportNoti = "ReportNoti" + DiffNoti = "DiffNoti" + CustomNoti = "CustomNoti" + NotiFile = "NotiFile" + WebHookNoti = "WebHookNoti" + // noti for telegram + TeleMess = "TeleMess" + TeleMessWrap = "TeleMessWrap" + TeleMessByFile = "TeleMessByFile" + TeleSendFile = "TeleSendFile" +) + +const ( + ExecCmd = "ExecCmd" + ExecCmdB = "ExecCmdB" + ExecCmdWithOutput = "ExecCmdWithOutput" + ExecContain = "ExecContain" + Sleep = "Sleep" + Exit = "Exit" + CastToInt = "CastToInt" + StripSlash = "StripSlash" + Printf = "Printf" + Cat = "Cat" + SortU = "SortU" + SplitFile = "SplitFile" + Append = "Append" + Copy = "Copy" + CreateFolder = "CreateFolder" + DeleteFile = "DeleteFile" + DeleteFolder = "DeleteFolder" + SplitFileByPart = "SplitFileByPart" + FileLength = "FileLength" + IsFile = "IsFile" + EmptyDir = "EmptyDir" + EmptyFile = "EmptyFile" + ReadLines = "ReadLines" + Compress = "Compress" + Decompress = "Decompress" +) + +const ( + TotalSubdomain = "TotalSubdomain" + TotalDns = "TotalDns" + TotalScreenShot = "TotalScreenShot" + TotalTech = "TotalTech" + TotalVulnerability = "TotalVulnerability" + TotalArchive = "TotalArchive" + TotalLink = "TotalLink" + TotalDirb = "TotalDirb" + CreateReport = "CreateReport" +) + +const ( + RRSync = "RRSync" + Clone = "Clone" + FClone = "FClone" + PushResult = "PushResult" + PushFolder = "PushFolder" + PullFolder = "PullFolder" + DiffCompare = "DiffCompare" + GitDiff = "GitDiff" + LoopGitDiff = "LoopGitDiff" + // for gitlab API only + CreateRepo = "CreateRepo" + DeleteRepo = "DeleteRepo" + DeleteRepoByPid = "DeleteRepoByPid" + ListProjects = "ListProjects" +) + +const ( + UploadToS3 = "UploadToS3" + DownloadFromS3 = "DownloadFromS3" + DownloadFile = "DownloadFile" + GenMarkdownReport = "GenMarkdownReport" +) + +const ( + SetVar = "SetVar" + SetOSVar = "SetOSVar" + GetOSEnv = "GetOSEnv" +) diff --git a/core/report.go b/core/report.go new file mode 100644 index 0000000..d0293d6 --- /dev/null +++ b/core/report.go @@ -0,0 +1,215 @@ +package core + +import ( + "fmt" + "os" + "path" + "path/filepath" + "strings" + + "github.com/Jeffail/gabs/v2" + "github.com/fatih/color" + "github.com/j3ssie/osmedeus/libs" + "github.com/j3ssie/osmedeus/utils" + "github.com/olekukonko/tablewriter" + "github.com/spf13/cast" +) + +func ListWorkspaces(options libs.Options) (content [][]string) { + workspaces, err := os.ReadDir(utils.NormalizePath(options.Env.WorkspacesFolder)) + if err != nil { + utils.ErrorF("Error reading workspaces folder: %s", err) + return content + } + + for _, ws := range workspaces { + if ws.IsDir() { + status := "unknown" + flowName := "unknown" + progress := "N/A" + wsFolder := path.Join(utils.NormalizePath(options.Env.WorkspacesFolder), ws.Name()) + + if utils.DirLength(wsFolder) == 0 { + continue + } + + if utils.FileExists(path.Join(wsFolder, "done")) { + status = "done" + } + + runtimeFile := path.Join(wsFolder, "runtime") + if utils.FileExists(runtimeFile) { + utils.DebugF("Reading information from: %v", runtimeFile) + runtimeContent := utils.GetFileContent(runtimeFile) + + if jsonParsed, ok := gabs.ParseJSON([]byte(runtimeContent)); ok == nil { + flowName = cast.ToString(jsonParsed.S("task_name").Data()) + doneStep := cast.ToString(jsonParsed.S("done_step").Data()) + totalSteps := cast.ToString(jsonParsed.S("total_steps").Data()) + isRunning := cast.ToString(jsonParsed.S("is_running").Data()) + + if isRunning == "true" { + status = "running" + } + + progress = color.HiCyanString(fmt.Sprintf(doneStep + "/" + totalSteps)) + } + } + + row := []string{ + path.Base(color.HiMagentaString(ws.Name())), status, flowName, progress, wsFolder, + } + content = append(content, row) + } + } + + table := tablewriter.NewWriter(os.Stderr) + table.SetAutoFormatHeaders(false) + table.SetHeader([]string{"Workspace Name", "Status", "Routine", "Progress", "Workspace Path"}) + table.SetBorders(tablewriter.Border{Left: true, Top: true, Right: true, Bottom: true}) + table.SetColWidth(120) + table.AppendBulk(content) + table.Render() + + fmt.Println(color.HiGreenString("šŸ“ Total Workspaces: ") + color.HiMagentaString("%v", len(content))) + usage := color.HiWhiteString("šŸ’” How to view report:") + color.HiGreenString(" osmedeus report view -t %v", color.HiMagentaString("[targetName]")) + fmt.Println(usage) + + return content +} + +func ListSingleWorkspace(options libs.Options, target string) (content [][]string) { + workspaces, err := os.ReadDir(utils.NormalizePath(options.Env.WorkspacesFolder)) + if err != nil { + utils.ErrorF("Error reading workspaces folder: %s", err) + return content + } + + header := []string{"Module", "Report Name", "Report Path"} + for _, ws := range workspaces { + if !ws.IsDir() { + continue + } + // compare target name with workspace name + if target != path.Base(ws.Name()) { + continue + } + wsFolder := path.Join(utils.NormalizePath(options.Env.WorkspacesFolder), ws.Name()) + + runtimeFile := path.Join(wsFolder, "runtime") + // only listing file that in report part + if utils.FileExists(runtimeFile) && !options.Report.Raw { + utils.InforF("Reading runtime information from: %v", runtimeFile) + runtimeContent := utils.GetFileContent(runtimeFile) + + isImported := false + if !strings.Contains(runtimeContent, options.Env.WorkspacesFolder) { + isImported = true + } + + // replace the workspace folder if it doesn't exist + if !strings.Contains(runtimeContent, options.Env.WorkspacesFolder) { + homeFolder := "/root/.osmedeus/workspaces/" + if strings.Contains(runtimeContent, "/root/workspaces-osmedeus/") { + homeFolder = "/root/workspaces-osmedeus/" + } + runtimeContent = strings.ReplaceAll(runtimeContent, homeFolder, options.Env.WorkspacesFolder+"/") + } + + if strings.Contains(runtimeContent, "/root/.osmedeus/workspaces") { + runtimeContent = strings.ReplaceAll(runtimeContent, "/root/.osmedeus/workspaces", options.Env.WorkspacesFolder) + } + + row := []string{"==> Workspace Name", color.HiGreenString(ws.Name()), color.HiGreenString(wsFolder)} + content = append(content, row) + + if jsonParsed, ok := gabs.ParseJSON([]byte(runtimeContent)); ok == nil { + reports := jsonParsed.S("target", "reports").Children() + + for _, report := range reports { + moduleName := cast.ToString(report.S("module").Data()) + reportName := cast.ToString(report.S("report_name").Data()) + reportPath := cast.ToString(report.S("report_path").Data()) + + if !utils.FileExists(reportPath) { + // /root/.osmedeus/workspaces + continue + } + + row := []string{ + moduleName, processReport(options, reportName), processReport(options, reportPath), + } + + content = append(content, row) + } + + markDownSunmmary := cast.ToString(jsonParsed.S("markdown_summary").Data()) + if utils.FileExists(markDownSunmmary) { + row := []string{"==> Markdown Summary", path.Base(markDownSunmmary), color.HiGreenString(markDownSunmmary)} + content = append(content, row) + } + markDownReport := cast.ToString(jsonParsed.S("markdown_report").Data()) + if utils.FileExists(markDownReport) { + row := []string{"==> Markdown HTML Summary", path.Base(markDownReport), color.HiGreenString(markDownReport)} + content = append(content, row) + } + + sep := []string{"==> --------", color.HiGreenString("----------"), color.HiGreenString("-----------")} + content = append(content, sep) + + if len(content) <= 2 && isImported { + utils.WarnF("Workspace folder not found in runtime file, you might extract it from different machine that being run on different user") + utils.WarnF("šŸ’” If you still have problem, please view it as raw format: %s", color.HiGreenString("osmedeus view -t %s --raw", target)) + } + + } + continue + } + + // --raw flag: list all avaliable file + header = []string{"Report Name", "Report Path"} + if options.Report.Static { + header = []string{"Report Name", "Report URL"} + } + row := []string{"==> Workspace Name", color.HiGreenString(ws.Name())} + content = append(content, row) + + filepath.Walk(wsFolder, func(reportPath string, _ os.FileInfo, err error) error { + reportName := path.Base(reportPath) + row := []string{processReport(options, reportName), processReport(options, reportPath)} + content = append(content, row) + return nil + }) + + sep := []string{"<== --------", color.HiGreenString("-----------")} + content = append(content, sep) + + } + + table := tablewriter.NewWriter(os.Stderr) + table.SetAutoFormatHeaders(true) + table.SetHeader(header) + table.SetBorders(tablewriter.Border{Left: true, Top: true, Right: true, Bottom: true}) + table.SetColWidth(120) + table.AppendBulk(content) + table.SetHeaderLine(true) + table.Render() + + return content +} + +func processReport(options libs.Options, reportPath string) string { + if options.Report.Static { + base := fmt.Sprintf("https://%s:8000/%s/workspaces", options.Report.PublicIP, options.Server.StaticPrefix) + reportPath = strings.ReplaceAll(reportPath, options.Env.WorkspacesFolder, base) + } + + if strings.HasSuffix(reportPath, ".html") { + reportPath = color.HiCyanString(reportPath) + } + if strings.HasSuffix(reportPath, ".json") { + reportPath = color.HiBlueString(reportPath) + } + + return reportPath +} diff --git a/core/runner.go b/core/runner.go new file mode 100644 index 0000000..70262e1 --- /dev/null +++ b/core/runner.go @@ -0,0 +1,373 @@ +package core + +import ( + "fmt" + "os" + "strings" + "sync" + + "github.com/Shopify/yaml" + "github.com/fatih/color" + "github.com/j3ssie/osmedeus/database" + "github.com/j3ssie/osmedeus/execution" + "github.com/j3ssie/osmedeus/libs" + "github.com/j3ssie/osmedeus/utils" + "github.com/panjf2000/ants" + "github.com/robertkrimen/otto" + "github.com/thoas/go-funk" +) + +// Runner runner struct to start a job +type Runner struct { + Input string + Workspace string + + InputType string // domain, url, ip, cidr or domain-file, url-file, ip-file, cidr-file + RequiredInput string // this should match with InputType + IsInvalid bool + ForceParams bool + + RoutineType string // module or flow + RoutineName string // general + RoutinePath string + RunnerSource string // cli or api + RunnerType string // local or cloud + + Opt libs.Options + + // use for analytics + DoneStep int + TotalSteps int + RunningTime int + CurrentModule string + + DoneFile string + RuntimeFile string + WorkspaceFolder string + + RoutineModules []string + Reports []string + Routines []libs.Routine + + VM *otto.Otto + TargetObj database.Target + ScanObj database.Scan + + Target map[string]string + // this is same as targets but won't change during the execution time + Params map[string]string +} + +// InitRunner init runner +func InitRunner(input string, opt libs.Options) (Runner, error) { + var runner Runner + runner.Input = input + runner.Opt = opt + runner.PrepareRoutine() + runner.InitVM() + + runner.RunnerSource = "cli" + // @TODO check if running in cloud + runner.RunnerType = "local" + return runner, nil +} + +// PrepareWorkflow prepare workflow file +func (r *Runner) PrepareWorkflow() { + allFlows := ListAllFlowName(r.Opt) + + r.RoutineType = "flow" + var err error + flows := SelectFlow(r.Opt.Scan.Flow, r.Opt) + for _, flow := range flows { + if strings.TrimSpace(flow) == "" { + continue + } + r.Opt.Flow, err = ParseFlow(flow) + r.RoutinePath = flow + if err != nil { + continue + } + + if r.Opt.Flow.NoDB { + r.Opt.NoDB = true + } + + if r.Opt.Flow.Type == "general" { + r.Opt.Flow.Input = r.Opt.Scan.Input + } + // default folder to look for module file if not specify + r.Opt.Flow.DefaultType = r.Opt.Flow.Type + + r.Target["FlowPath"] = flow + r.RequiredInput = r.Opt.Flow.Validator + r.ForceParams = r.Opt.Flow.ForceParams + + // get more params from flow + if len(r.Opt.Flow.Params) > 0 { + for _, params := range r.Opt.Flow.Params { + for k, v := range params { + r.Target[k] = v + } + } + } + } + + // generate routines + for _, routine := range r.Opt.Flow.Routines { + // select module depend on the flow type + if routine.FlowFolder != "" { + r.Opt.Flow.Type = routine.FlowFolder + } else { + r.Opt.Flow.Type = r.Opt.Flow.DefaultType + } + + modules := SelectModules(routine.Modules, r.Opt) + routine.RoutineName = fmt.Sprintf("flow-%s", r.Opt.Flow.Name) + + for _, module := range modules { + parsedModule, err := ParseModules(module) + if err != nil || parsedModule.Name == "" { + continue + } + r.TotalSteps += len(parsedModule.Steps) + routine.ParsedModules = append(routine.ParsedModules, parsedModule) + } + r.Routines = append(r.Routines, routine) + } + + if len(r.Routines) == 0 { + if r.Opt.Scan.Flow != "cloud-distributed" { + utils.WarnF("Your workflow %v doesn't exist", color.HiRedString(r.Opt.Scan.Flow)) + utils.WarnF("Please select one of these flow: %v", color.HiMagentaString(strings.Join(allFlows, ", "))) + } + } +} + +func (r *Runner) PrepareModule() { + allModules := ListModuleName(r.Opt) + r.RoutineType = "module" + + var err error + for _, rawModule := range r.Opt.Scan.Modules { + var routine libs.Routine + + module := DirectSelectModule(r.Opt, rawModule) + r.RoutinePath = rawModule + r.Opt.Module, err = ParseModules(module) + if err != nil || r.Opt.Module.Name == "" { + utils.WarnF("Your module %v doesn't exist", color.HiRedString(r.Opt.Scan.Modules[0])) + utils.WarnF("Please select one of these module: %v", color.HiMagentaString(strings.Join(allModules, ", "))) + continue + } + if r.Opt.Module.NoDB { + r.Opt.NoDB = true + } + + r.Target["FlowPath"] = "direct-module" + routine.ParsedModules = append(routine.ParsedModules, r.Opt.Module) + routine.RoutineName = fmt.Sprintf("module-%s", r.Opt.Flow.Name) + r.Target["Module"] = module + + r.Routines = append(r.Routines, routine) + + r.TotalSteps += len(r.Opt.Module.Steps) + r.RequiredInput = r.Opt.Module.Validator + } +} + +func (r *Runner) PrepareRoutine() { + // prepare targets + r.Target = ParseInput(r.Input, r.Opt) + r.Workspace = r.Target["Workspace"] + + // take from -m flag + if len(r.Opt.Scan.Modules) > 0 { + r.RoutineName = strings.Join(r.Opt.Scan.Modules, "-") + r.PrepareModule() + return + } + + // take from -f flag + if r.Opt.Scan.Flow != "" { + r.RoutineName = r.Opt.Scan.Flow + r.PrepareWorkflow() + } +} + +// PrepareParams prepare global params +func (r *Runner) PrepareParams() { + r.Params = r.Target + + // looking for more params from each module + for _, routine := range r.Routines { + for _, module := range routine.ParsedModules { + // params from module file + if len(module.Params) > 0 { + for _, param := range module.Params { + for k, v := range param { + // skip params if override: false + _, exist := r.Params[k] + if r.ForceParams && exist { + utils.DebugF("Skip override param: %v --> %v", k, v) + continue + } + + v = ResolveData(v, r.Params) + if strings.HasPrefix(v, "~/") { + v = utils.NormalizePath(v) + } + r.Params[k] = v + } + } + } + + if len(r.Opt.Scan.ParamsFile) > 0 { + var params map[string]string + yamlFile, err := os.ReadFile(r.Opt.Scan.ParamsFile) + if err != nil { + utils.ErrorF("YAML parsing err: %v -- #%v ", r.Opt.Scan.ParamsFile, err) + return + } + err = yaml.Unmarshal(yamlFile, ¶ms) + if err != nil { + utils.ErrorF("Error unmarshal: %v -- %v", params, err) + return + } + if len(params) > 0 { + for k, v := range params { + v = ResolveData(v, r.Params) + r.Params[k] = v + } + } + } + + // more params from -p flag which will override everything + if len(r.Opt.Scan.Params) > 0 { + params := ParseParams(r.Opt.Scan.Params) + if len(params) > 0 { + for k, v := range params { + v = ResolveData(v, r.Params) + r.Params[k] = v + } + } + } + } + } + + r.ResolveRoutine() + +} + +// ResolveRoutine resolve the module name first +func (r *Runner) ResolveRoutine() { + var routines []libs.Routine + + for _, rawRoutine := range r.Routines { + + var routine libs.Routine + for _, module := range rawRoutine.ParsedModules { + module = ResolveReports(module, r.Params) + + r.Reports = append(r.Reports, module.Report.Final...) + module.PreRun = ResolveSlice(module.PreRun, r.Params) + + // steps + for i, step := range module.Steps { + module.Steps[i].Timeout = ResolveData(step.Timeout, r.Params) + module.Steps[i].Threads = ResolveData(step.Threads, r.Params) + module.Steps[i].Label = ResolveData(step.Label, r.Params) + module.Steps[i].Std = ResolveData(step.Std, r.Params) + module.Steps[i].Source = ResolveData(step.Source, r.Params) + + module.Steps[i].Conditions = ResolveSlice(step.Conditions, r.Params) + module.Steps[i].Required = ResolveSlice(step.Required, r.Params) + + module.Steps[i].Commands = ResolveSlice(step.Commands, r.Params) + module.Steps[i].Scripts = ResolveSlice(step.Scripts, r.Params) + + module.Steps[i].RCommands = ResolveSlice(step.RCommands, r.Params) + module.Steps[i].RScripts = ResolveSlice(step.RScripts, r.Params) + module.Steps[i].PConditions = ResolveSlice(step.PConditions, r.Params) + module.Steps[i].PScripts = ResolveSlice(step.PScripts, r.Params) + module.Steps[i].Ose = ResolveSlice(step.Ose, r.Params) + } + + module.PostRun = ResolveSlice(module.PostRun, r.Params) + routine.ParsedModules = append(routine.ParsedModules, module) + } + + routines = append(routines, routine) + } + r.Routines = routines +} + +func (r *Runner) Start() { + err := r.Validator() + if err != nil { + utils.ErrorF("Input does not match the require type: %v -- %v", r.RequiredInput, r.Input) + utils.InforF("Adding %v flag if you want to disable input validate", color.HiCyanString(`'--nv'`)) + return + } + utils.InforF("Running %s tactic with baseline threads hold as %s", color.YellowString(r.Opt.Tactics), color.HiMagentaString("%v", r.Opt.Threads)) + + r.Opt.Scan.ROptions = r.Target + // prepare some metadata files + utils.MakeDir(r.Target["Output"]) + r.DoneFile = r.Target["Output"] + "/done" + r.RuntimeFile = r.Target["Output"] + "/runtime" + r.WorkspaceFolder = r.Target["Output"] + os.Remove(r.DoneFile) + + utils.InforF("Running the routine %v on %v", color.HiYellowString(r.RoutineName), color.CyanString(r.Input)) + utils.InforF("Detailed runtime file can be found on %v", color.CyanString(r.RuntimeFile)) + execution.TeleSendMess(r.Opt, fmt.Sprintf("%s -- Start new scan: %s -- %s", r.Opt.Noti.ClientName, r.Opt.Scan.Flow, r.Target["Workspace"]), "#status", false) + + r.DBNewTarget() + r.DBNewScan() + r.LoadEngineScripts() + + r.PrepareParams() + + ///// + /* really start the scan here */ + r.StartRoutines() + ///// + + r.DBDoneScan() + utils.BlockF("Finished", fmt.Sprintf("The scan for %v was completed within %v", color.HiCyanString(r.Input), color.HiMagentaString("%vs", r.RunningTime))) + + if r.Opt.EnableBackup { + r.BackupWorkspace() + } +} + +// StartRoutines start the scan +func (r *Runner) StartRoutines() { + for _, routine := range r.Routines { + // start each section of modules + r.RunRoutine(routine.ParsedModules) + } +} + +func (r *Runner) RunRoutine(modules []libs.Module) { + var wg sync.WaitGroup + p, _ := ants.NewPoolWithFunc(r.Opt.Concurrency*10, func(m interface{}) { + module := m.(libs.Module) + r.RunModule(module) + wg.Done() + }, ants.WithPreAlloc(true)) + defer p.Release() + + for _, module := range modules { + if funk.ContainsString(r.Opt.Exclude, module.Name) { + utils.BadBlockF("Module-Excluded", fmt.Sprintf("Module %v has been excluded", color.CyanString(module.Name))) + continue + } + + p.Invoke(module) + wg.Add(1) + } + + wg.Wait() +} diff --git a/core/runtime.go b/core/runtime.go new file mode 100644 index 0000000..0c5db00 --- /dev/null +++ b/core/runtime.go @@ -0,0 +1,488 @@ +package core + +import ( + "fmt" + "os" + "strings" + "time" + + "github.com/fatih/color" + "github.com/j3ssie/osmedeus/execution" + "github.com/j3ssie/osmedeus/utils" + "github.com/robertkrimen/otto" + "github.com/spf13/cast" +) + +// InitVM init scripting engine +func (r *Runner) InitVM() { + r.VM = otto.New() + r.LoadEngineScripts() +} + +func (r *Runner) ExecScript(script string) string { + utils.DebugF("[Run-Scripts] %v", script) + value, err := r.VM.Run(script) + if err == nil { + out, nerr := value.ToString() + if nerr == nil { + return out + } + } + + return "" +} + +// RunOse really start the runner +func (r *Runner) RunOse(scriptName string) { + scriptContent := scriptName + if !strings.Contains(scriptName, "\n") { + scriptFile := SelectScript(scriptName, r.Opt) + if utils.FileExists(scriptFile) { + scriptContent = utils.GetFileContent(scriptFile) + } + } + + if len(scriptContent) == 0 { + utils.ErrorF("Error running script: %s", scriptName) + return + } + + utils.DebugF("-- Start ose:\n\n%s", scriptName) + r.ExecScript(scriptContent) + utils.DebugF("-- Done ose: %s", scriptName) +} + +func (r *Runner) ConditionExecScript(script string) bool { + utils.DebugF("[Run-Scripts] %v", script) + value, err := r.VM.Run(script) + + if err == nil { + out, nerr := value.ToBoolean() + if nerr == nil { + return out + } + } + + return false +} + +func (r *Runner) LoadEngineScripts() { + r.LoadScripts() + r.LoadDBScripts() + // r.LoadImportScripts() + r.LoadExternalScripts() + r.LoadGitScripts() + r.LoadNotiScripts() +} + +func (r *Runner) LoadScripts() string { + var output string + vm := r.VM + + // set attribute + vm.Set("Target", r.Target) + + // SetVar('length', 6) + vm.Set(SetVar, func(call otto.FunctionCall) otto.Value { + args := call.ArgumentList + varName := args[0].String() + value := args[1].String() + r.Target[varName] = value + return otto.Value{} + }) + + // Exit script used to exit the module + vm.Set(Exit, func(call otto.FunctionCall) otto.Value { + args := call.ArgumentList + output = fmt.Sprintf("exit(%v)", args[0].String()) + utils.InforF("Exit Detected") + result, err := vm.ToValue(output) + if err == nil { + return result + } + return otto.Value{} + }) + + // ExecCmd execute command + vm.Set(ExecCmd, func(call otto.FunctionCall) otto.Value { + cmd := call.Argument(0).String() + _, err := utils.RunCommandWithErr(cmd) + var validate bool + if err != nil { + validate = true + } + result, err := vm.ToValue(validate) + if err != nil { + return otto.Value{} + } + return result + }) + + // Cat the file to stdout + vm.Set(Cat, func(call otto.FunctionCall) otto.Value { + filename := call.Argument(0).String() + utils.InforF("Showing the content of: %v", color.HiCyanString(filename)) + utils.Cat(filename) + result, err := vm.ToValue(true) + if err != nil { + return otto.Value{} + } + return result + }) + + // ExecCmdB execute in the background + vm.Set(ExecCmdB, func(call otto.FunctionCall) otto.Value { + cmd := call.Argument(0).String() + go func() { + utils.RunOSCommand(cmd) + }() + result, _ := vm.ToValue(true) + return result + }) + + // ExecCmd execute command + vm.Set(ExecCmdWithOutput, func(call otto.FunctionCall) otto.Value { + utils.RunCommandSteamOutput(call.Argument(0).String()) + result, err := vm.ToValue(true) + if err != nil { + return otto.Value{} + } + return result + }) + + // ExecCmd execute command + vm.Set(ExecContain, func(call otto.FunctionCall) otto.Value { + out := utils.RunCmdWithOutput(call.Argument(0).String()) + expected := call.Argument(2).String() + validate := strings.Contains(out, expected) + result, err := vm.ToValue(validate) + if err != nil { + return otto.Value{} + } + return result + }) + + // CastToInt convert string to int + vm.Set(CastToInt, func(call otto.FunctionCall) otto.Value { + toInt := cast.ToInt(call.Argument(0).String()) + result, err := vm.ToValue(toInt) + if err == nil { + return result + } + return otto.Value{} + }) + + vm.Set(FileLength, func(call otto.FunctionCall) otto.Value { + data := utils.FileLength(call.Argument(0).String()) + utils.DebugF("FileLength -- %v", data) + result, err := vm.ToValue(data) + if err == nil { + return result + } + return otto.Value{} + }) + + vm.Set(IsFile, func(call otto.FunctionCall) otto.Value { + data := utils.FileLength(call.Argument(0).String()) + var validate bool + if data > 1 { + validate = true + } + result, _ := vm.ToValue(validate) + return result + }) + + // StripSlash strip last '/' of URL + vm.Set(StripSlash, func(call otto.FunctionCall) otto.Value { + raw := call.Argument(0).String() + out := strings.Trim(raw, "/") + result, err := vm.ToValue(out) + if err != nil { + return otto.Value{} + } + return result + }) + + vm.Set(ReadLines, func(call otto.FunctionCall) otto.Value { + fileName := call.Argument(0).String() + data := utils.ReadingLines(fileName) + if len(data) > 0 { + result, err := vm.ToValue(data) + if err == nil { + return result + } + } + return otto.Value{} + }) + + // Printf simply print a string to console + vm.Set(Printf, func(call otto.FunctionCall) otto.Value { + fmt.Printf("%v\n", color.HiCyanString(call.Argument(0).String())) + returnValue, _ := otto.ToValue(true) + return returnValue + }) + + // split file to multiple + vm.Set(SplitFile, func(call otto.FunctionCall) otto.Value { + execution.SplitFile("size", call.ArgumentList) + return otto.Value{} + }) + + // split file to multiple + vm.Set(SplitFileByPart, func(call otto.FunctionCall) otto.Value { + execution.SplitFile("part", call.ArgumentList) + return otto.Value{} + }) + + vm.Set(Sleep, func(call otto.FunctionCall) otto.Value { + execution.Sleep(call.Argument(0).String()) + returnValue, _ := otto.ToValue(true) + return returnValue + }) + + vm.Set(SortU, func(call otto.FunctionCall) otto.Value { + execution.SortU(call.Argument(0).String()) + returnValue, _ := otto.ToValue(true) + return returnValue + }) + + vm.Set(Append, func(call otto.FunctionCall) otto.Value { + dest := call.Argument(0).String() + src := call.Argument(1).String() + execution.Append(dest, src) + returnValue, _ := otto.ToValue(true) + return returnValue + }) + + vm.Set(Decompress, func(call otto.FunctionCall) otto.Value { + dest := call.Argument(0).String() + src := call.Argument(1).String() + execution.Decompress(dest, src) + returnValue, _ := otto.ToValue(true) + return returnValue + }) + + vm.Set(Compress, func(call otto.FunctionCall) otto.Value { + dest := call.Argument(0).String() + src := call.Argument(1).String() + execution.Compress(dest, src) + returnValue, _ := otto.ToValue(true) + return returnValue + }) + + vm.Set(CreateFolder, func(call otto.FunctionCall) otto.Value { + utils.MakeDir(call.Argument(0).String()) + return otto.Value{} + }) + + vm.Set(DeleteFile, func(call otto.FunctionCall) otto.Value { + execution.DeleteFile(call.Argument(0).String()) + return otto.Value{} + }) + + vm.Set(DeleteFolder, func(call otto.FunctionCall) otto.Value { + execution.DeleteFolder(call.Argument(0).String()) + return otto.Value{} + }) + + vm.Set(Copy, func(call otto.FunctionCall) otto.Value { + src := call.Argument(0).String() + dest := call.Argument(1).String() + execution.Copy(src, dest) + return otto.Value{} + }) + + vm.Set(GetOSEnv, func(call otto.FunctionCall) otto.Value { + args := call.ArgumentList + env := args[0].String() + defaultValue := env + if len(args) > 1 { + defaultValue = args[1].String() + } + utils.GetOSEnv(env, defaultValue) + return otto.Value{} + }) + + vm.Set(SetOSVar, func(call otto.FunctionCall) otto.Value { + args := call.ArgumentList + varName := strings.ToUpper(args[0].String()) + defaultValue := varName + if len(args) > 1 { + defaultValue = args[1].String() + } + err := os.Setenv(varName, defaultValue) + if err != nil { + utils.ErrorF("Error setting environment variable: %v", err) + } + return otto.Value{} + }) + + vm.Set(EmptyDir, func(call otto.FunctionCall) otto.Value { + result, _ := vm.ToValue(utils.EmptyDir(call.Argument(0).String())) + return result + }) + + vm.Set(EmptyFile, func(call otto.FunctionCall) otto.Value { + result, err := vm.ToValue(utils.EmptyFile(call.Argument(0).String(), 0)) + if err != nil { + return otto.Value{} + } + if len(call.ArgumentList) > 1 { + num, _ := call.Argument(0).ToInteger() + result, err = vm.ToValue(utils.EmptyFile(call.Argument(0).String(), int(num))) + if err != nil { + return otto.Value{} + } + } + return result + }) + + vm.Set(RRSync, func(call otto.FunctionCall) otto.Value { + vpsIP := call.Argument(0).String() // root@ipaddress + src := call.Argument(1).String() // local path + dest := call.Argument(2).String() // remote path + cmd := fmt.Sprintf("rsync -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s' -avzr --progress %s %s:%s", r.Opt.Cloud.SecretKey, src, vpsIP, dest) + r.RetryCommandWithExpectString(cmd, `bytes/sec`) + return otto.Value{} + }) + + r.VM = vm + + return output +} + +func (r *Runner) RetryCommandWithExpectString(cmd string, expectString string, timeoutRaw ...string) string { + timeout := "300s" + if len(timeoutRaw) > 0 { + timeout = timeoutRaw[0] + } + var out string + + utils.DebugF("Retry command: %s", cmd) + for i := 0; i < r.Opt.Cloud.Retry; i++ { + if timeout == "000" { + out, _ = utils.RunOSCommand(cmd) + } else { + out = utils.RunCmdWithOutput(cmd, timeout) + } + + if !strings.Contains(out, expectString) { + utils.DebugF(out) + time.Sleep(time.Duration(60*(i+1)) * time.Second) + continue + } + return out + } + return out +} + +func (r *Runner) LoadNotiScripts() string { + var output string + vm := r.VM + options := r.Opt + + // script for notification + vm.Set(StartNoti, func(call otto.FunctionCall) otto.Value { + execution.StatusNoti("start", options) + return otto.Value{} + }) + vm.Set(DoneNoti, func(call otto.FunctionCall) otto.Value { + execution.StatusNoti("done", options) + return otto.Value{} + }) + vm.Set(ReportNoti, func(call otto.FunctionCall) otto.Value { + execution.ReportNoti(call.ArgumentList, options) + return otto.Value{} + }) + vm.Set(DiffNoti, func(call otto.FunctionCall) otto.Value { + execution.DiffNoti(call.ArgumentList, options) + return otto.Value{} + }) + // CustomNoti("message here") + vm.Set(CustomNoti, func(call otto.FunctionCall) otto.Value { + execution.SendAttachment("custom", call.Argument(0).String(), options) + return otto.Value{} + }) + // NotiFile("src") + vm.Set(NotiFile, func(call otto.FunctionCall) otto.Value { + execution.SendFile(call.Argument(0).String(), options.Noti.SlackReportChannel, options) + return otto.Value{} + }) + // using webhook + vm.Set(WebHookNoti, func(call otto.FunctionCall) otto.Value { + args := call.ArgumentList + messContent := args[0].String() + messType := "custom" + if len(args) > 1 { + messType = args[0].String() + messContent = args[1].String() + } + + execution.WebHookSendAttachment(options, messType, messContent) + return otto.Value{} + }) + + // Telegram functions + + vm.Set(TeleMess, func(call otto.FunctionCall) otto.Value { + args := call.ArgumentList + messContent := args[0].String() + channelType := "general" + if len(args) > 1 { + channelType = args[0].String() + messContent = args[1].String() + } + execution.TeleSendMess(options, messContent, channelType, false) + return otto.Value{} + }) + // send message but with inside ``` + vm.Set(TeleMessWrap, func(call otto.FunctionCall) otto.Value { + args := call.ArgumentList + messContent := args[0].String() + channelType := "general" + if len(args) > 1 { + channelType = args[0].String() + messContent = args[1].String() + } + execution.TeleSendMess(options, messContent, channelType, true) + return otto.Value{} + }) + + vm.Set(TeleMessByFile, func(call otto.FunctionCall) otto.Value { + args := call.ArgumentList + fileName := args[0].String() + channelType := "general" + if len(args) > 1 { + channelType = args[0].String() + fileName = args[1].String() + } + if !utils.FileExists(fileName) { + utils.DebugF("File %s not found", fileName) + return otto.Value{} + } + + messContent := utils.GetFileContent(fileName) + if len(messContent) > 4000 { + execution.TeleSendFile(options, fileName, channelType) + } else { + execution.TeleSendMess(options, messContent, channelType, true) + } + + return otto.Value{} + }) + + vm.Set(TeleSendFile, func(call otto.FunctionCall) otto.Value { + args := call.ArgumentList + messContent := args[0].String() + channelType := "general" + if len(args) > 1 { + channelType = args[0].String() + messContent = args[1].String() + } + execution.TeleSendFile(options, messContent, channelType) + return otto.Value{} + }) + + return output + +} diff --git a/core/step.go b/core/step.go new file mode 100644 index 0000000..97fe556 --- /dev/null +++ b/core/step.go @@ -0,0 +1,167 @@ +package core + +import ( + "context" + "fmt" + "strings" + "sync" + "time" + + "github.com/fatih/color" + "github.com/j3ssie/osmedeus/libs" + "github.com/j3ssie/osmedeus/utils" + "github.com/thoas/go-funk" +) + +func (r *Runner) RunModulesWithTimeout(timeoutRaw string, module libs.Module, options libs.Options) { + timeout := utils.CalcTimeout(timeoutRaw) + utils.InforF("Run module %v with %v seconds timeout", color.HiCyanString(module.Name), timeout) + + c := context.Background() + deadline := time.Now().Add(time.Duration(timeout) * time.Second) + c, cancel := context.WithDeadline(c, deadline) + defer cancel() + + go func() { + r.RunModule(module) + cancel() + }() + + select { + case <-c.Done(): + utils.DebugF("Module done") + return + case <-time.After(time.Duration(timeout) * time.Second): + utils.BadBlockF("timeout", fmt.Sprintf("Module got timeout after %v", color.HiMagentaString(timeoutRaw))) + } + return +} + +// CheckResume check resume report +func CheckResume(module libs.Module) bool { + for _, report := range module.Report.Final { + if !strings.Contains(report, ".osmedeus/storages") && !utils.FileExists(report) { + return false + } + } + return true +} + +// ResolveReports resolve real path of reports +func ResolveReports(module libs.Module, params map[string]string) libs.Module { + var final []string + var noti []string + var diff []string + for _, report := range module.Report.Final { + final = append(final, ResolveData(report, params)) + } + + for _, report := range module.Report.Noti { + noti = append(noti, ResolveData(report, params)) + } + + for _, report := range module.Report.Diff { + diff = append(diff, ResolveData(report, params)) + } + + module.Report.Final = final + module.Report.Noti = noti + module.Report.Diff = diff + return module +} + +// print all report +func printReports(module libs.Module) { + var files []string + files = append(files, module.Report.Final...) + files = append(files, module.Report.Noti...) + files = append(files, module.Report.Diff...) + + reports := funk.UniqString(files) + utils.BlockF("Report", color.HiCyanString("List of reports generated by the %v module", color.HiGreenString(module.Name))) + for _, report := range reports { + if !utils.FileExists(report) && utils.EmptyFile(report, 0) { + if !utils.FolderExists(report) && utils.EmptyDir(report) { + continue + } + } + utils.BlockF("report-file", report) + } +} + +// CheckRequired check if required file exist or not +func (r *Runner) CheckRequired(requires []string) error { + if len(requires) == 0 { + return nil + } + + utils.DebugF("Checking requirements: %v", requires) + for _, require := range requires { + + if strings.Contains(require, "(") && strings.Contains(require, ")") { + validate := r.ConditionExecScript(require) + if !validate { + utils.DebugF("Requirement not met: %v", require) + return fmt.Errorf("condition not met: %s", require) + } + continue + } + + require = utils.NormalizePath(require) + if !utils.FileExists(require) && utils.EmptyFile(require, 0) { + if !utils.FolderExists(require) && utils.DirLength(require) == 0 { + utils.DebugF("Missing %v", require) + return fmt.Errorf("missing requirement") + } + } + } + return nil +} + +// CheckCondition check if required file exist or not +func (r *Runner) CheckCondition(conditions []string) error { + if len(conditions) == 0 { + return nil + } + for _, require := range conditions { + if !r.ConditionExecScript(require) { + return fmt.Errorf("condition not met: %s", require) + } + } + return nil +} + +// RunCommands run list of commands in parallel +func (r *Runner) RunCommands(commands []string, std string) string { + var wg sync.WaitGroup + var output string + var err error + + for _, command := range commands { + wg.Add(1) + // don't run too much command at once + go func(command string) { + defer wg.Done() + var out string + if std != "" { + out, err = utils.RunOSCommand(command) + } else { + err = utils.RunCommandWithoutOutput(command) + } + + if err != nil { + utils.DebugF("error running command: %v -- %v", command, err) + } + + if out != "" { + output += out + } + }(command) + } + wg.Wait() + + if std != "" { + utils.WriteToFile(std, output) + } + return output +} diff --git a/core/tmux.go b/core/tmux.go new file mode 100644 index 0000000..47afe7b --- /dev/null +++ b/core/tmux.go @@ -0,0 +1,85 @@ +package core + +import ( + "fmt" + "strings" + + "github.com/j3ssie/osmedeus/libs" + "github.com/j3ssie/osmedeus/utils" +) + +type Tmux struct { + ApplyAll bool + SelectedWindow string + Exclude string + Limit int + Windows []string +} + +func InitTmux(options libs.Options) (Tmux, error) { + cmd := "tmux ls" + var tmux Tmux + tmux.ApplyAll = options.Tmux.ApplyAll + tmux.SelectedWindow = options.Tmux.SelectedWindow + tmux.Exclude = options.Tmux.Exclude + tmux.Limit = options.Tmux.Limit + + raw := utils.RunCmdWithOutput(cmd) + if strings.Contains(raw, "command not found") || !strings.Contains(raw, "\n") { + return tmux, fmt.Errorf("tmux program not installed") + } + + stds := strings.Split(raw, "\n") + for _, line := range stds { + if strings.TrimSpace(line) == "" || !strings.Contains(line, " ") { + continue + } + + data := strings.Split(line, " ") + tmux.Windows = append(tmux.Windows, strings.TrimRight(data[0], ":")) + } + return tmux, nil +} + +func (t *Tmux) ListTmux() { + if len(t.Windows) == 0 { + fmt.Println("No tmux available") + return + } + fmt.Println(strings.Join(t.Windows, ", ")) +} + +func (t *Tmux) CatchSession() string { + var result string + if len(t.Windows) == 0 { + fmt.Println("No tmux available") + return result + } + for _, window := range t.Windows { + utils.DebugF("Get info of: %s", window) + if !t.ApplyAll { + if window != t.SelectedWindow { + continue + } + } + if strings.HasPrefix(window, t.Exclude) { + continue + } + utils.InforF("Get output of %s session", window) + cmd := fmt.Sprintf(`tmux capture-pane -pt "%s"`, window) + raw := utils.RunCmdWithOutput(cmd) + + data := strings.Split(raw, "\n") + if t.Limit == 0 { + fmt.Println(raw) + } else if t.Limit < len(data) && t.Limit > 0 { + fmt.Println(strings.Join(data[len(data)-t.Limit:len(data)-1], "\n")) + } else { + fmt.Println(raw) + } + + result += raw + } + + return result +} diff --git a/core/token.go b/core/token.go new file mode 100644 index 0000000..23213c9 --- /dev/null +++ b/core/token.go @@ -0,0 +1,288 @@ +package core + +import ( + "fmt" + "os" + "path" + "strings" + + "github.com/fatih/color" + "github.com/j3ssie/osmedeus/execution" + "github.com/j3ssie/osmedeus/libs" + "github.com/j3ssie/osmedeus/utils" + "github.com/spf13/viper" +) + +func SetupOSEnv(options *libs.Options) { + utils.DebugF("Loading all environment variables from: %v", options.TokenConfigFile) + v = viper.New() + v.SetConfigName("osm-var") + v.SetConfigType("yaml") + v.AddConfigPath(path.Dir(options.TokenConfigFile)) + + // Read the configuration file + err := v.ReadInConfig() + if err != nil { + v.SetDefault("Storages", map[string]string{ + // path of secret key for push result + // ~/.osmedeus/secret/storages_key + "secret_key": utils.GetOSEnv("SECRET_KEY", "SECRET_KEY"), + // the repo format should be like this "git@gitlab.com:j3ssie/example.git", + "summary_storage": path.Join(options.Env.RootFolder, "storages/summary"), + "summary_repo": utils.GetOSEnv("SUMMARY_REPO", "SUMMARY_REPO"), + "subdomain_storage": path.Join(options.Env.RootFolder, "storages/subdomain"), + "subdomain_repo": utils.GetOSEnv("SUBDOMAIN_REPO", "SUBDOMAIN_REPO"), + "assets_storage": path.Join(options.Env.RootFolder, "storages/assets"), + "assets_repo": utils.GetOSEnv("ASSETS_REPO", "ASSETS_REPO"), + "ports_storage": path.Join(options.Env.RootFolder, "storages/ports"), + "ports_repo": utils.GetOSEnv("PORTS_REPO", "PORTS_REPO"), + "http_storage": path.Join(options.Env.RootFolder, "storages/http"), + "http_repo": utils.GetOSEnv("HTTP_REPO", "HTTP_REPO"), + "vuln_storage": path.Join(options.Env.RootFolder, "storages/vuln"), + "vuln_repo": utils.GetOSEnv("VULN_REPO", "VULN_REPO"), + "paths_storage": path.Join(options.Env.RootFolder, "storages/paths"), + "paths_repo": utils.GetOSEnv("PATHS_REPO", "PATHS_REPO"), + "mics_storage": path.Join(options.Env.RootFolder, "storages/mics"), + "mics_repo": utils.GetOSEnv("MICS_REPO", "MICS_REPO"), + }) + + // dedicated storages + v.SetDefault("Git", map[string]string{ + "base_url": utils.GetOSEnv("GITLAB_BASE_URL", "https://gitlab.com"), + "api": utils.GetOSEnv("GITLAB_API_TOKEN", "GITLAB_API_TOKEN"), + "username": utils.GetOSEnv("GITLAB_USER", "GITLAB_USER"), + "password": utils.GetOSEnv("GITLAB_PASS", "GITLAB_PASS"), + "group": utils.GetOSEnv("GITLAB_GROUP", "GITLAB_GROUP"), + "prefix_name": utils.GetOSEnv("GITLAB_PREFIX_NAME", "deosm"), + "default_tag": utils.GetOSEnv("GITLAB_DEFAULT_TAG", "osmd"), + "default_user": utils.GetOSEnv("GITLAB_DEFAULT_USER", "j3ssie"), + "default_uid": utils.GetOSEnv("GITLAB_DEFAULT_UID", "3537075"), + "destorage": path.Join(options.Env.RootFolder, "destorage"), + }) + + v.SetDefault("Notification", map[string]string{ + "client_name": utils.GetOSEnv("CLIENT_NAME", "CLIENT_NAME"), + "slack_status_channel": utils.GetOSEnv("SLACK_STATUS_CHANNEL", "SLACK_STATUS_CHANNEL"), + "slack_report_channel": utils.GetOSEnv("SLACK_REPORT_CHANNEL", "SLACK_REPORT_CHANNEL"), + "slack_diff_channel": utils.GetOSEnv("SLACK_DIFF_CHANNEL", "SLACK_DIFF_CHANNEL"), + "slack_webhook": utils.GetOSEnv("SLACK_WEBHOOK", "SLACK_WEBHOOK"), + "telegram_channel": utils.GetOSEnv("TELEGRAM_CHANNEL", "TELEGRAM_CHANNEL"), + "telegram_status_channel": utils.GetOSEnv("TELEGRAM_STATUS_CHANNEL", "TELEGRAM_STATUS_CHANNEL"), + "telegram_report_channel": utils.GetOSEnv("TELEGRAM_REPORT_CHANNEL", "TELEGRAM_REPORT_CHANNEL"), + "telegram_sensitive_channel": utils.GetOSEnv("TELEGRAM_SENSITIVE_CHANNEL", "TELEGRAM_SENSITIVE_CHANNEL"), + "telegram_dirb_channel": utils.GetOSEnv("TELEGRAM_DIRB_CHANNEL", "TELEGRAM_DIRB_CHANNEL"), + "telegram_mics_channel": utils.GetOSEnv("TELEGRAM_MICS_CHANNEL", "TELEGRAM_MICS_CHANNEL"), + }) + + v.SetDefault("Cdn", map[string]string{ + "cdn_s3_bucket": utils.GetOSEnv("CDN_S3_BUCKET", "CDN_S3_BUCKET"), + "cdn_aws_access_key": utils.GetOSEnv("CDN_AWS_ACCESS_KEY", "CDN_AWS_ACCESS_KEY"), + "cdn_aws_secret_key": utils.GetOSEnv("CDN_AWS_SECRET_KEY", "CDN_AWS_SECRET_KEY"), + "cdn_aws_region": utils.GetOSEnv("CDN_AWS_REGION", "ap-southeast-1"), + }) + + // default tokens but you're feel free to add more and it will be automatically loaded to ENV + v.SetDefault("Tokens", map[string]string{ + "slack": utils.GetOSEnv("SLACK_API_TOKEN", "SLACK_API_TOKEN"), + "telegram": utils.GetOSEnv("TELEGRAM_API_TOKEN", "TELEGRAM_API_TOKEN"), + "gitlab": utils.GetOSEnv("GITLAB_API_TOKEN", "GITLAB_API_TOKEN"), + "github": utils.GetOSEnv("GITHUB_API_KEY", "GITHUB_API_KEY"), + }) + + if ok := v.WriteConfigAs(options.TokenConfigFile); ok != nil { + utils.ErrorF("Error writing config file: %s", ok) + } + utils.InforF("Created a new token configuration file at %s", color.HiCyanString(options.ConfigFile)) + + } + + if err := v.ReadInConfig(); err != nil { + utils.ErrorF("Error reading config file, %s", err) + } + // Read tokens as a list of maps + tokens := v.GetStringMapString("tokens") + if len(tokens) > 0 { + utils.DebugF("Adding %v tokens to the environment variables", color.HiMagentaString("%v", len(tokens))) + // Iterate through each token + for name, value := range tokens { + // automatic convert to upper case + name = strings.ToUpper(name) + + // skip if the value is equal to the name + if name == value { + continue + } + + utils.DebugF("Setting environment variable: %v -- %v", name, value) + + err := os.Setenv(name, value) + if err != nil { + utils.ErrorF("Error setting environment variable: %v -- %v", name, err) + } + } + } + // get all the config that need to be set manually + GetStorages(options) + GetStorages(options) + GetNotification(options) + GetGit(options) + GetCdn(options) +} + +func GetStorages(options *libs.Options) { + if !options.PremiumPackage || utils.GetOSEnv("ENABLE_GIT_STORAGES", "") != "TRUE" { + return + } + + storages := v.GetStringMapString("Storages") + + // get variables from config.yaml + storagesOptions := make(map[string]string) + for k, dest := range storages { + storages[k] = utils.NormalizePath(dest) + storagesOptions[k] = utils.NormalizePath(dest) + } + secretKey := storages["secret_key"] + if secretKey == "" || secretKey == "SECRET_KEY" { + return + } + + storagesOptions["secret_key"] = secretKey + // load default existing key if it exists + defaultKey := path.Join(options.Env.BaseFolder, "secret/storages_key") + if !utils.FileExists(secretKey) && utils.FileExists(defaultKey) { + utils.InforF("Loaded default secret for storages from: %v", color.HiCyanString(defaultKey)) + if _, err := utils.RunCommandWithErr(fmt.Sprintf("cp %s %s && chmod 600 %s", defaultKey, secretKey, secretKey)); err != nil { + utils.ErrorF("error copying default secret key: %v", defaultKey) + } + } + + if !utils.FileExists(secretKey) { + utils.InforF("No SSH key for storages found. Generate a new one at: %v", color.HiCyanString(secretKey)) + if _, err := utils.RunCommandWithErr(fmt.Sprintf(`ssh-keygen -t ed25519 -f %s -q -N ''`, secretKey)); err != nil { + color.Red("[-] error generated SSH Key for storages at: %v", secretKey) + return + } + utils.InforF("Please add the public key at %v to your gitlab profile", color.HiCyanString(secretKey+".pub")) + } + + if !utils.FileExists("~/.gitconfig") { + utils.WarnF("Looks like you didn't set up the git user at %v yet", color.HiCyanString("~/.gitconfig")) + utils.WarnF("šŸ’” Init git info with this command: %s", color.HiCyanString(`git config --global user.name "your-username" && git config --global user.email "your-username@users.noreply.gitlab.com"`)) + } + + if options.CustomGit { + // in case custom repo is set + for _, env := range os.Environ() { + // the ENV should be OSM_SUMMARY_STORAGE, OSM_SUMMARY_REPO + if strings.HasSuffix(env, "OSM_") { + data := strings.Split(env, "=") + key := strings.ToLower(data[0]) + value := strings.Replace(env, data[0]+"=", "", -1) + + if strings.HasSuffix(key, "summary_storage") { + storagesOptions["summary_storage"] = value + } + if strings.HasSuffix(key, "summary_repo") { + storagesOptions["summary_repo"] = value + } + + if strings.HasSuffix(key, "assets_storage") { + storagesOptions["assets_storage"] = value + } + if strings.HasSuffix(key, "assets_repo") { + storagesOptions["assets_repo"] = value + } + + if strings.HasSuffix(key, "ports_storage") { + storagesOptions["ports_storage"] = value + } + if strings.HasSuffix(key, "ports_repo") { + storagesOptions["ports_repo"] = value + } + } + } + } + + storagesOptions[storages["summary_storage"]] = storages["summary_repo"] + storagesOptions[storages["subdomain_storage"]] = storages["subdomain_repo"] + storagesOptions[storages["http_storage"]] = storages["http_repo"] + storagesOptions[storages["assets_storage"]] = storages["assets_repo"] + storagesOptions[storages["mics_storage"]] = storages["mics_repo"] + storagesOptions[storages["ports_storage"]] = storages["ports_repo"] + storagesOptions[storages["paths_storage"]] = storages["paths_repo"] + storagesOptions[storages["vuln_storage"]] = storages["vuln_repo"] + options.Storages = storagesOptions + + // disable git feature or no secret key found + if options.NoGit { + options.Storages["secret_key"] = "" + } + if options.Storages["secret_key"] == "" { + options.NoGit = true + } + + if options.NoGit { + return + } + execution.CloneRepo(storages["summary_repo"], storages["summary_storage"], *options) + execution.CloneRepo(storages["http_repo"], storages["http_storage"], *options) + execution.CloneRepo(storages["assets_repo"], storages["assets_storage"], *options) + execution.CloneRepo(storages["subdomain_repo"], storages["subdomain_storage"], *options) + execution.CloneRepo(storages["ports_repo"], storages["ports_storage"], *options) + execution.CloneRepo(storages["mics_repo"], storages["mics_storage"], *options) + execution.CloneRepo(storages["paths_repo"], storages["paths_storage"], *options) + execution.CloneRepo(storages["vuln_repo"], storages["vuln_storage"], *options) +} + +// GetNotification get storge repos +func GetNotification(options *libs.Options) { + noti := v.GetStringMapString("Notification") + tokens := v.GetStringMapString("Tokens") + + // tokens + options.Noti.SlackToken = tokens["slack"] + options.Noti.TelegramToken = tokens["telegram"] + + // this mean you're not setup the notification yet + if len(options.Noti.TelegramToken) < 20 { + options.NoNoti = true + } + + options.Noti.ClientName = noti["client_name"] + + options.Noti.SlackStatusChannel = noti["slack_status_channel"] + options.Noti.SlackReportChannel = noti["slack_report_channel"] + options.Noti.SlackDiffChannel = noti["slack_diff_channel"] + options.Noti.SlackWebHook = noti["slack_webhook"] + options.Noti.TelegramChannel = noti["telegram_channel"] + options.Noti.TelegramSensitiveChannel = noti["telegram_sensitive_channel"] + options.Noti.TelegramReportChannel = noti["telegram_report_channel"] + options.Noti.TelegramStatusChannel = noti["telegram_status_channel"] + options.Noti.TelegramDirbChannel = noti["telegram_dirb_channel"] + options.Noti.TelegramMicsChannel = noti["telegram_mics_channel"] +} + +// GetCdn get options for client +func GetCdn(options *libs.Options) { + cdn := v.GetStringMapString("Cdn") + options.Cdn.Bucket = cdn["cdn_s3_bucket"] + options.Cdn.AccessKeyId = cdn["cdn_aws_access_key"] + options.Cdn.SecretKey = cdn["cdn_aws_secret_key"] + options.Cdn.Region = cdn["cdn_aws_region"] +} + +// GetGit get options for client +func GetGit(options *libs.Options) { + git := v.GetStringMapString("Git") + options.Git.BaseURL = git["base_url"] + options.Git.DeStorage = git["destorage"] + options.Git.Token = git["api"] + options.Git.Username = git["username"] + options.Git.Password = git["password"] + options.Git.Group = git["group"] + options.Git.DefaultPrefix = git["prefix_name"] + options.Git.DefaultTag = git["default_tag"] + options.Git.DefaultUser = git["default_user"] + options.Git.DefaultUID = utils.StrToInt(git["default_uid"]) +} diff --git a/core/update.go b/core/update.go new file mode 100644 index 0000000..6854438 --- /dev/null +++ b/core/update.go @@ -0,0 +1,336 @@ +package core + +import ( + "fmt" + "os" + "os/exec" + "path" + "strings" + "time" + + "github.com/cenkalti/backoff/v4" + "github.com/fatih/color" + "github.com/hashicorp/go-version" + "github.com/j3ssie/osmedeus/libs" + "github.com/j3ssie/osmedeus/provider" + "github.com/j3ssie/osmedeus/utils" + jsoniter "github.com/json-iterator/go" + "github.com/mitchellh/go-homedir" +) + +/* Mostly calling OS commands for double-check the PATH too */ + +func CheckUpdate(options *libs.Options) bool { + // t.Format("02-Jan-2006") + + var shouldUpdate bool + // ~/.osmedeus/update/metadata.json + options.Update.UpdateConfig = path.Join(utils.NormalizePath(options.Env.RootFolder), "update") + utils.MakeDir(options.Update.UpdateConfig) + + if options.Update.MetaDataURL == "" { + options.Update.MetaDataURL = fmt.Sprintf("%s/public.json", libs.METADATA) + if options.PremiumPackage { + options.Update.MetaDataURL = fmt.Sprintf("%s/premium.json", libs.METADATA) + } + } + + utils.InforF("Checking metadata information from: %v", options.Update.MetaDataURL) + metadataFile := path.Join(options.Update.UpdateConfig, "metadata.json") + + var oldMetaData libs.UpdateMetaData + oldMetaData.CoreVersion = libs.VERSION + oldMetaData.WorkflowVersion = "v0.0.1" + + if utils.FileExists(metadataFile) { + oldMetaDataContent := utils.GetFileContent(metadataFile) + if err := jsoniter.UnmarshalFromString(oldMetaDataContent, &oldMetaData); err != nil { + utils.ErrorF("error to parse metadata: %v", metadataFile) + return false + } + } + + var newMetaData libs.UpdateMetaData + res := utils.SendGET("", options.Update.MetaDataURL) + if res.StatusCode == 200 { + if err := jsoniter.UnmarshalFromString(res.Body, &newMetaData); err != nil { + utils.ErrorF("error to parse metadata: %v", options.Update.MetaDataURL) + return false + } + + utils.InforF("Writing metadata to: %v", color.HiCyanString(metadataFile)) + if data, err := jsoniter.MarshalToString(&newMetaData); err == nil { + utils.WriteToFile(metadataFile, data) + } + } else { + utils.ErrorF("error fetching metadata from: %v", options.Update.MetaDataURL) + return false + } + + utils.DebugF(res.Body) + + v1, err := version.NewVersion(oldMetaData.CoreVersion) + if err != nil { + utils.ErrorF("error parsing version: %v -- %v", oldMetaData.CoreVersion, err) + return false + } + + // get from metadata URL + v2, err := version.NewVersion(newMetaData.CoreVersion) + if err != nil { + utils.ErrorF("error parsing version: %v -- %v", newMetaData.CoreVersion, err) + + return false + } + + // Comparison example. There is also GreaterThan, Equal, and just + // a simple Compare that returns an int allowing easy >=, <=, etc. + if v1.LessThan(v2) { + fmt.Printf("Your current %s %s are outdated. Latest is %s\n", libs.BINARY, color.HiMagentaString("%v", v1), color.HiGreenString("%v", v2)) + shouldUpdate = true + } else { + fmt.Printf("You're using %s core latest version %s updated at %s\n", libs.BINARY, color.HiMagentaString("%v", v1), color.HiGreenString("%v", newMetaData.UpdatedAt)) + } + + // check workflow version if core is updated + if !shouldUpdate { + wfv1, err := version.NewVersion(oldMetaData.WorkflowVersion) + if err != nil { + utils.ErrorF("error parsing version: %v -- %v", oldMetaData.WorkflowVersion, err) + } + wfv2, err := version.NewVersion(newMetaData.WorkflowVersion) + if err != nil { + utils.ErrorF("error parsing version: %v -- %v", newMetaData.WorkflowVersion, err) + } + if wfv1.LessThan(wfv2) { + fmt.Printf("Your current %s workflow %s are outdated. Latest is %s\n", libs.BINARY, color.HiMagentaString("%v", v1), color.HiGreenString("%v", v2)) + shouldUpdate = true + } + } + + if shouldUpdate { + home, _ := homedir.Dir() + fmt.Printf("šŸ“– Run %s again to update Check out this page for more detail: %s\n", color.HiGreenString("the same install script"), color.HiGreenString("https://docs.osmedeus.org/installation/")) + fmt.Printf("šŸ’” If you want a fresh install please run the command: %s\n", color.HiBlueString("rm -rf %s/osmedeus-base %s/.osmedeus", home, home)) + } + + return shouldUpdate +} + +func GetUpdateURL(options libs.Options) string { + if !options.PremiumPackage { + utils.InforF("Getting update url of public community package") + return libs.INSTALL + } + utils.InforF("šŸ’Ž Getting update url of premium package") + + providerConfigs, err := provider.ParseProvider(options.CloudConfigFile) + if err != nil { + utils.ErrorF("error to parse provider config: %v", err) + return "" + } + + return providerConfigs.Builder.BuildRepo +} + +func RunUpdate(options libs.Options) error { + if options.Update.UpdateURL == "" { + return fmt.Errorf("no update URL") + } + + if options.Update.CleanOldData { + utils.InforF("Cleaning old data: %s", color.HiCyanString(options.Env.BaseFolder)) + os.RemoveAll(options.Env.BaseFolder) + } + + if options.PremiumPackage { + utils.InforF("Running update from premium package install script") + } else { + utils.InforF("Running update from: %v", color.HiCyanString(options.Update.UpdateURL)) + } + + options.Update.UpdateScript = fmt.Sprintf("/tmp/%s-update.sh", libs.BINARY) + cmd := fmt.Sprintf("rm -rf %s && wget -qO %s %s ", options.Update.UpdateScript, options.Update.UpdateScript, options.Update.UpdateURL) + _, err := utils.RunCommandWithErr(cmd) + if err != nil { + utils.ErrorF("error to run update script: %v", err) + return err + } + + if !utils.FileExists(options.Update.UpdateScript) { + utils.ErrorF("update script doesn't exist: %v", options.Update.UpdateScript) + return fmt.Errorf("update script doesn't exist: %v", options.Update.UpdateScript) + } + + if _, err := utils.RunCommandSteamOutput(fmt.Sprintf(`bash %s`, options.Update.UpdateScript)); err != nil { + utils.ErrorF("error to run update script: %v", err) + return err + } + return nil +} + +func GenerateMetaData(options libs.Options) { + utils.InforF("Generating metadata to: %v", color.HiCyanString(options.Update.GenerateMeta)) + t := time.Now() + t.Format("2006-01-02T15:04:05") + var updateData = libs.UpdateMetaData{ + CoreVersion: libs.VERSION, + WorkflowVersion: libs.VERSION, + UpdatedAt: t.Format("2006-01-02T15:04"), + } + + if data, ok := jsoniter.MarshalToString(updateData); ok == nil { + utils.InforF("Generate meta data: %v", data) + utils.WriteToFile(options.Update.GenerateMeta, data) + } +} + +func GitUpdate(opt libs.Options) error { + cmd := fmt.Sprintf("GIT_SSH_COMMAND='ssh -o StrictHostKeyChecking=no -i %v' git clone --depth=1 %v %v", opt.Storages["secret_key"], opt.Update.UpdateURL, opt.Update.UpdateFolder) + _, err := utils.RunCommandWithErr(cmd) + return err +} + +func HTTPUpdate(opt libs.Options) error { + cmd := fmt.Sprintf("wget -q %s -O %s", opt.Update.UpdateURL, opt.Update.UpdateFolder) + _, err := utils.RunCommandWithErr(cmd) + return err +} + +func DownloadUpdate(opt libs.Options) error { + os.RemoveAll(opt.Update.UpdateFolder) + utils.InforF("Downloading the update folder via %v: %v", opt.Update.UpdateType, opt.Update.UpdateURL) + var err error + + backOff := backoff.NewExponentialBackOff() + backOff.MaxElapsedTime = 1200 * time.Second + backOff.Multiplier = 2.0 + backOff.InitialInterval = 30 * time.Second + + operation := func() error { + switch strings.ToLower(opt.Update.UpdateType) { + case "git": + err = GitUpdate(opt) + case "s3", "http": + err = HTTPUpdate(opt) + default: + err = GitUpdate(opt) + } + if err != nil { + utils.ErrorF("error downloading update content: %s -- %s", opt.Update.UpdateType, opt.Update.UpdateURL) + } + return err + } + err = backoff.Retry(operation, backOff) + if err != nil { + utils.ErrorF("error downloading update content: %s -- %s", opt.Update.UpdateType, opt.Update.UpdateURL) + return err + } + return nil +} + +func Update(opt libs.Options) { + os.RemoveAll(opt.Update.UpdateFolder) + utils.MakeDir(opt.Update.UpdateFolder) + + updateScript := fmt.Sprintf("%s/update.sh", opt.Update.UpdateFolder) + cmd := fmt.Sprintf("wget -q %s -O %s/install.sh", opt.Update.UpdateURL, updateScript) + if _, err := utils.RunCommandWithErr(cmd); err != nil { + utils.ErrorF("error downloading the update script: %v", opt.Update.UpdateURL) + return + } + + cmd = fmt.Sprintf("bash %s", updateScript) + if _, err := utils.RunCommandWithErr(cmd); err != nil { + utils.ErrorF("error running the update script: %v", updateScript) + return + } +} + +func UpdateVuln(opt libs.Options) { + utils.InforF("Updating Vulnerability Database only") + + // update nuclei templates + utils.DebugF("Updating Nuclei Templates") + nucleiTemplate := utils.NormalizePath("~/nuclei-templates") + + if utils.DirLength(nucleiTemplate) > 0 { + os.RemoveAll(nucleiTemplate) + } + utils.RunOSCommand(fmt.Sprintf("git clone --depth=1 https://github.com/projectdiscovery/nuclei-templates.git %s", nucleiTemplate)) +} + +func UpdateBase(opt libs.Options) { + err := DownloadUpdate(opt) + if err != nil { + return + } + + // change the folder since we will update it + if opt.Update.IsUpdateBin { + utils.InforF("Updating External binaries") + binPath := path.Join(opt.Update.UpdateFolder, "binaries") + utils.Move(binPath, opt.Env.BinariesFolder) + opt.Update.UpdateFolder = path.Join(opt.Update.UpdateFolder, fmt.Sprintf("%s-base", libs.BINARY)) + } + + // update Env + utils.InforF("Updating Environments Data") + utils.Move(path.Join(opt.Update.UpdateFolder, "data"), opt.Env.DataFolder) + utils.Move(path.Join(opt.Update.UpdateFolder, "workflow"), opt.Env.WorkFlowsFolder) + utils.Move(path.Join(opt.Update.UpdateFolder, "ose"), opt.Env.OseFolder) + utils.Move(path.Join(opt.Update.UpdateFolder, "ui"), opt.Env.UIFolder) + + utils.Move(path.Join(opt.Update.UpdateFolder, "clouds"), opt.Env.CloudConfigFolder) + os.Chmod(opt.Cloud.SecretKey, 0600) + + // update osmedeus core binary + corePath, err := exec.LookPath(libs.BINARY) + utils.InforF("Updating %v binary at %v", color.HiCyanString(libs.BINARY), color.HiCyanString(corePath)) + if err == nil { + os.RemoveAll(corePath) + newBin := fmt.Sprintf("%s/dist/%s", strings.TrimRight(opt.Update.UpdateFolder, "/"), libs.BINARY) + unZipCmd := fmt.Sprintf("unzip %s/dist/%s-linux.zip -d %s/dist/", strings.TrimRight(opt.Update.UpdateFolder, "/"), libs.BINARY, strings.TrimRight(opt.Update.UpdateFolder, "/")) + utils.RunOSCommand(unZipCmd) + + // update binaries in gopath + goPath := utils.GetOSEnv("GOPATH", "GOPATH") + if goPath != "GOPATH" { + goPath = path.Join(goPath, fmt.Sprintf("bin/%s", libs.BINARY)) + os.RemoveAll(goPath) + utils.RunOSCommand(fmt.Sprintf("cp %s %s", newBin, goPath)) + + // go path but in plugins folder + goPath = path.Join(opt.Env.BinariesFolder, "go", libs.BINARY) + os.RemoveAll(goPath) + utils.RunOSCommand(fmt.Sprintf("cp %s %s", newBin, goPath)) + } + utils.Move(newBin, corePath) + } + + // update vulnerability signatures + utils.InforF("Updating Jaeles Signatures") + jaelesSign := path.Join(opt.Update.UpdateFolder, "pro-signatures") + if utils.DirLength(jaelesSign) > 0 { + utils.RunOSCommand(fmt.Sprintf("jaeles config reload --signDir %s", jaelesSign)) + utils.Move(jaelesSign, "~/pro-signatures") + } else { + os.RemoveAll(utils.NormalizePath("~/pro-signatures")) + utils.RunOSCommand(fmt.Sprintf("GIT_SSH_COMMAND='ssh -o StrictHostKeyChecking=no -i %s' git clone --depth=1 git@gitlab.com:j3ssie/pro-signatures ~/pro-signatures", opt.Storages["secret_key"])) + utils.RunOSCommand(fmt.Sprintf("rm -rf ~/custom-nuclei-template && GIT_SSH_COMMAND='ssh -o StrictHostKeyChecking=no -i %s' git clone --depth=1 git@gitlab.com:j3ssie/custom-nuclei-template.git ~/custom-nuclei-template", opt.Storages["secret_key"])) + utils.RunOSCommand("jaeles config reload --signDir ~/pro-signatures") + } + + // update nuclei templates + utils.InforF("Updating Nuclei Templates") + nucleiTemplate := path.Join(opt.Update.UpdateFolder, "nuclei-templates") + if utils.DirLength(nucleiTemplate) > 0 { + utils.Move(nucleiTemplate, utils.NormalizePath("~/nuclei-templates")) + } else { + utils.RunOSCommand(fmt.Sprintf("git clone --depth=1 https://github.com/projectdiscovery/nuclei-templates.git ~/nuclei-templates")) + } + + // clean up + utils.InforF("Clean up update folder") + os.RemoveAll(opt.Update.UpdateFolder) +} diff --git a/core/validate.go b/core/validate.go new file mode 100644 index 0000000..e16e801 --- /dev/null +++ b/core/validate.go @@ -0,0 +1,144 @@ +package core + +import ( + "fmt" + "path" + "strings" + + "github.com/fatih/color" + "github.com/go-playground/validator/v10" + "github.com/j3ssie/osmedeus/libs" + "github.com/j3ssie/osmedeus/utils" +) + +func (r *Runner) Validator() error { + if r.RequiredInput == "" || r.Opt.DisableValidateInput { + return nil + } + + r.RequiredInput = strings.ToLower(strings.TrimSpace(r.RequiredInput)) + var inputAsFile bool + // cidr, cidr-file + if strings.HasSuffix(r.RequiredInput, "-file") || r.RequiredInput == "file" { + inputAsFile = true + } + v := validator.New() + + // if input as a file + if utils.FileExists(r.Input) && inputAsFile { + r.InputType = "file" + inputs := utils.ReadingLines(r.Input) + + for index, input := range inputs { + if strings.TrimSpace(input) == "" { + continue + } + // no more validation if it's file 'validator: file' + if r.RequiredInput == "file" { + continue + } + + inputType, err := validate(v, input) + // fmt.Println("r.RequiredInput, inputType", r.RequiredInput, inputType) + if err == nil { + // really validate thing + if !strings.HasPrefix(r.RequiredInput, inputType) { + utils.DebugF("validate: %v -- %v", input, inputType) + errString := fmt.Sprintf("line %v in %v file not match the require input: %v -- %v", index, r.Input, input, inputType) + utils.ErrorF(errString) + return fmt.Errorf(errString) + } + } + } + return nil + + } + + var err error + r.InputType, err = validate(v, r.Input) + if err != nil { + utils.ErrorF("unrecognized input: %v", r.Input) + return err + } + utils.InforF("Start validating input: %v -- %v", color.HiCyanString(r.Input), color.HiCyanString(r.InputType)) + + if !strings.HasPrefix(r.RequiredInput, r.InputType) { + return fmt.Errorf("input does not match the require validation: inputType:%v -- requireType:%v", r.InputType, r.RequiredInput) + } + + if inputAsFile { + utils.MakeDir(libs.TEMP) + suffix := utils.RandomString(4) + if r.Opt.Scan.SuffixName != "" { + suffix = r.Opt.Scan.SuffixName + "-" + utils.RandomString(4) + } + dest := path.Join(libs.TEMP, fmt.Sprintf("%v-%v", utils.StripPath(r.Input), suffix)) + if r.Opt.Scan.CustomWorkspace != "" { + dest = path.Join(libs.TEMP, fmt.Sprintf("%v-%v", utils.StripPath(r.Opt.Scan.CustomWorkspace), suffix)) + } + utils.WriteToFile(dest, r.Input) + utils.InforF("Convert input to a file: %v", dest) + r.Input = dest + r.Target = ParseInput(r.Input, r.Opt) + } + + utils.DebugF("validator: input:%v -- type: %v -- require:%v", r.Input, r.InputType, r.RequiredInput) + return nil +} + +func validate(v *validator.Validate, raw string) (string, error) { + var err error + var inputType string + + if utils.FileExists(raw) { + inputType = "file" + } + + err = v.Var(raw, "required,url") + if err == nil { + inputType = "url" + } + + err = v.Var(raw, "required,ipv4") + if err == nil { + inputType = "ip" + } + + err = v.Var(raw, "required,fqdn") + if err == nil { + inputType = "domain" + } + + err = v.Var(raw, "required,hostname") + if err == nil { + inputType = "domain" + } + + err = v.Var(raw, "required,cidr") + if err == nil { + inputType = "cidr" + } + + err = v.Var(raw, "required,uri") + if err == nil { + inputType = "url" + } + + err = v.Var(raw, "required,uri") + if err == nil { + inputType = "url" + if strings.HasPrefix(raw, "https://github.com") || strings.HasPrefix(raw, "https://gitlab.com") { + inputType = "git-url" + } + } + + if strings.HasPrefix(raw, "git@") { + inputType = "git-url" + } + + if inputType == "" { + return "", fmt.Errorf("unrecognized input") + } + + return inputType, nil +} diff --git a/core/validate_test.go b/core/validate_test.go new file mode 100644 index 0000000..b11364d --- /dev/null +++ b/core/validate_test.go @@ -0,0 +1,53 @@ +package core + +import ( + "fmt" + "github.com/j3ssie/osmedeus/libs" + "testing" +) + +func TestValidator(t *testing.T) { + input := "target.com" + opt := libs.Options{} + runner, _ := InitRunner(input, opt) + runner.RequiredInput = "domain" + runner.Validator() + + runner.Input = "apple.com" + runner.Validator() + + fmt.Printf("runner.InputType --> %v:%v -- %s\n", runner.RequiredInput, runner.InputType, runner.Input) + + runner.Input = "1.2.3.4" + runner.Validator() + + runner.Input = "http://127.0.0.1/q" + runner.Validator() + runner.Input = "sub.domain.com" + runner.Validator() + + runner.Input = "1.2.3.4/24" + runner.Validator() + + runner.Input = "https://github.com/j3ssie/osmedeus" + runner.Validator() + fmt.Printf("==> runner.InputType --> %v:%v -- %s\n\n", runner.RequiredInput, runner.InputType, runner.Input) + + runner.Input = "git@github.com:j3ssie/osmedeus.git" + runner.Validator() + fmt.Printf("==> runner.InputType --> %v:%v -- %s\n\n", runner.RequiredInput, runner.InputType, runner.Input) + + // + ////raw := "tcp://git@gitlab.com:j3ssie/osmd-assets" + //raw := "git@gitlab.com/j3ssie/osmd-assets" + //v := validator.New() + //err := v.Var(raw, "required,uri") + //fmt.Println(err) + // + //err = v.Var(raw, "required,datauri") + //fmt.Println(err) + + if runner.InputType == "" { + t.Errorf("Error Validator") + } +} diff --git a/database/connect.go b/database/connect.go new file mode 100644 index 0000000..2697d4f --- /dev/null +++ b/database/connect.go @@ -0,0 +1,63 @@ +package database + +// load driver + +// // DB gorm connector +// var DB *gorm.DB + +// // InitDB connect to db +// func InitDB(options libs.Options) (*gorm.DB, error) { +// newLogger := logger.New( +// log.New(ioutil.Discard, "\r\n", log.LstdFlags), // io writer +// logger.Config{ +// SlowThreshold: time.Second, // Slow SQL threshold +// LogLevel: logger.Info, // Log level +// IgnoreRecordNotFoundError: true, // Ignore ErrRecordNotFound error for logger +// Colorful: false, // Disable color +// }, +// ) + +// config := gorm.Config{ +// SkipDefaultTransaction: false, +// NamingStrategy: nil, +// FullSaveAssociations: false, +// Logger: newLogger, +// NowFunc: nil, +// DryRun: false, +// PrepareStmt: false, +// DisableAutomaticPing: false, +// DisableForeignKeyConstraintWhenMigrating: false, +// DisableNestedTransaction: false, +// AllowGlobalUpdate: false, +// QueryFields: false, +// CreateBatchSize: 0, +// ClauseBuilders: nil, +// ConnPool: nil, +// Dialector: nil, +// Plugins: nil, +// } + +// var err error +// if options.Server.DBType == "mysql" { +// utils.InforF("Connect Database at %v:%v", options.Server.DBHost, options.Server.DBPort) +// utils.InforF("Use Database: %v", options.Server.DBName) +// DB, err = gorm.Open(mysql.Open(options.Server.DBConnection), &config) +// } else { +// DB, err = gorm.Open(sqlite.Open(options.Server.DBPath), &config) +// } + +// if err != nil { +// fmt.Printf("Error Database connect at %v:%v -- %v\n", options.Server.DBHost, options.Server.DBPort, err) +// return nil, err +// } + +// // scanning data +// // DB.AutoMigrate(&User{}) +// //DB.AutoMigrate(&Org{}) +// DB.AutoMigrate(&Target{}) +// DB.AutoMigrate(&Scan{}) +// DB.AutoMigrate(&Report{}) +// DB.AutoMigrate(&CloudInstance{}) +// // asset data +// return DB, nil +// } diff --git a/database/models.go b/database/models.go new file mode 100644 index 0000000..9a892fe --- /dev/null +++ b/database/models.go @@ -0,0 +1,99 @@ +package database + +import ( + "time" +) + +type Model struct { + CreatedAt time.Time `json:"created_at,omitempty"` + UpdatedAt time.Time `json:"updated_at,omitempty"` +} + +// Schedule store task to do every single time +type Schedule struct { + Model + TaskName string `gorm:"type:varchar(255)" json:"task_name"` + RefreshSeconds string `gorm:"type:varchar(255)" json:"refresh_seconds"` + Command string `gorm:"type:longtext" json:"command"` + Status string `gorm:"type:varchar(255)" json:"status"` +} + +/////////// + +type Scan struct { + Model + + // input part + InputName string `gorm:"type:varchar(255);not null" json:"input_name"` + InputType string `gorm:"type:varchar(255);default:'general'" json:"input_type"` + + TaskName string `gorm:"type:varchar(255)" json:"task_name"` + TaskPath string `gorm:"type:varchar(255)" json:"task_path"` + TaskType string `gorm:"type:varchar(255);default:'flow'" json:"task_type"` + + MarkDownSunmmary string `gorm:"type:varchar(255)" json:"markdown_summary"` + MarkDownReport string `gorm:"type:varchar(255)" json:"markdown_report"` + + RunningTime int `json:"running_time"` // as seconds + CurrentModule string `gorm:"type:varchar(255)" json:"current_module"` + DoneStep int `json:"done_step"` + TotalSteps int `json:"total_steps"` + + // mics part + LogFile string `json:"log_file"` + ProcessID int `json:"process_id"` + + // progress checking + IsRunning bool `json:"is_running"` + IsDone bool `json:"is_done"` + IsNew bool `json:"is_new"` + IsError bool `json:"is_error"` + IsStarted bool `json:"is_started"` + + // if the task is running by cloud provider + IsPrepared bool `json:"is_prepared"` + IsCloud bool `json:"is_cloud"` + CloudInfo string `json:"cloud_info"` + + Target Target `json:"target"` +} + +// runtime object +type Target struct { + InputName string `gorm:"type:varchar(255);unique;not null" json:"input_name"` + // @NOTE: below field shouldn't be show in UI + // Workspace == InputName but strip out '/' + Workspace string `gorm:"type:varchar(255);unique;not null" json:"workspace"` + InputType string `gorm:"type:varchar(255);default:'N/A'" json:"input_type"` + + // total number for stat + TotalAssets int `json:"total_assets"` + TotalDns int `json:"total_dns"` + TotalTech int `json:"total_tech"` + TotalScreenShot int `json:"total_screenshot"` + TotalVulnerability int `json:"total_vulnerability"` + TotalDirb int `json:"total_dirb"` + TotalLink int `json:"total_link"` + TotalArchive int `json:"total_archive"` + TotalIPRange int `json:"total_ip_range"` + TotalCloud int `json:"total_cloud"` + TotalCred int `json:"total_cred"` + + // flag information + IsNew bool `json:"is_new"` + IsWildCard bool `json:"is_wildcard"` + + Reports []Report `json:"reports"` +} + +// Report store reports file record +type Report struct { + ReportName string `gorm:"type:varchar(255)" json:"report_name"` + ReportPath string `gorm:"type:longtext" json:"report_path"` + + Module string `gorm:"type:varchar(255)" json:"module"` + ModulePath string `gorm:"type:longtext" json:"module_path"` + + WorkspaceName string `gorm:"type:varchar(255)" json:"workspace_name"` + ReportType string `gorm:"type:varchar(255);default:'text'" json:"report_type"` +} diff --git a/database/select.go b/database/select.go new file mode 100644 index 0000000..c462eb6 --- /dev/null +++ b/database/select.go @@ -0,0 +1,100 @@ +package database + +import ( + "os" + "path" + "path/filepath" + "strings" + + "github.com/j3ssie/osmedeus/libs" + "github.com/j3ssie/osmedeus/utils" + jsoniter "github.com/json-iterator/go" +) + +func GetAllWorkspaces(opt libs.Options) (directories []string) { + // Open the specified directory + dir, err := os.Open(opt.Env.WorkspacesFolder) + if err != nil { + return directories + } + defer dir.Close() + + // Read all entries in the directory + entries, err := dir.Readdir(-1) + if err != nil { + return directories + } + + for _, entry := range entries { + wsName := entry.Name() + directories = append(directories, wsName) + } + + return directories +} + +func GetAllScan(opt libs.Options) (scans []Scan) { + wss := GetAllWorkspaces(opt) + for _, wsName := range wss { + runtimeFile := filepath.Join(opt.Env.WorkspacesFolder, wsName, "runtime") + if !utils.FileExists(runtimeFile) { + continue + } + + // parse the content + runtimeContent := utils.GetFileContent(runtimeFile) + wsData := Scan{} + if err := jsoniter.UnmarshalFromString(runtimeContent, &wsData); err == nil { + + // replace the filepath with static prefix + wsData.MarkDownReport = strings.ReplaceAll(wsData.MarkDownReport, opt.Env.WorkspacesFolder, path.Join("/", opt.Server.StaticPrefix, "workspaces")) + wsData.MarkDownSunmmary = strings.ReplaceAll(wsData.MarkDownSunmmary, opt.Env.WorkspacesFolder, path.Join("/", opt.Server.StaticPrefix, "workspaces")) + scans = append(scans, wsData) + } + + } + return scans +} + +func GetSingleScan(wsName string, opt libs.Options) (scan Scan) { + runtimeFile := filepath.Join(opt.Env.WorkspacesFolder, wsName, "runtime") + if !utils.FileExists(runtimeFile) { + return scan + } + + // parse the content + runtimeContent := utils.GetFileContent(runtimeFile) + if err := jsoniter.UnmarshalFromString(runtimeContent, &scan); err == nil { + // replace the filepath with static prefix + scan.MarkDownReport = strings.ReplaceAll(scan.MarkDownReport, opt.Env.WorkspacesFolder, path.Join("/", opt.Server.StaticPrefix, "workspaces")) + scan.MarkDownSunmmary = strings.ReplaceAll(scan.MarkDownSunmmary, opt.Env.WorkspacesFolder, path.Join("/", opt.Server.StaticPrefix, "workspaces")) + return scan + } + return scan +} + +func GetScanProgress(opt libs.Options) (scans []Scan) { + rawScans := GetAllScan(opt) + + for _, scan := range rawScans { + scan.Target = Target{} + scans = append(scans, scan) + } + + return scans +} + +// func GetWorkspaceDetail(wsName string, opt libs.Options) (workspace Scan) { +// runtimeFile := filepath.Join(opt.Env.WorkspacesFolder, wsName, "runtime") +// if !utils.FileExists(runtimeFile) { +// return workspace +// } + +// // parse the content +// runtimeContent := utils.GetFileContent(runtimeFile) +// if err := jsoniter.UnmarshalFromString(runtimeContent, &workspace); err == nil { +// return workspace +// } + +// return workspace +// } diff --git a/distribute/clean.go b/distribute/clean.go new file mode 100644 index 0000000..0bfaa3f --- /dev/null +++ b/distribute/clean.go @@ -0,0 +1,54 @@ +package distribute + +import ( + "os" + "sync" + + "github.com/fatih/color" + "github.com/j3ssie/osmedeus/libs" + "github.com/j3ssie/osmedeus/utils" + "github.com/jinzhu/copier" + "github.com/panjf2000/ants" +) + +func ClearAllInstances(opt libs.Options) { + if opt.Cloud.Retry == 0 { + opt.Cloud.Retry = 8 + } + + // select all cloud instance + instances := GetAllInstances(opt) + if len(instances) == 0 { + utils.WarnF("no active cloud instance running") + return + } + + var wg sync.WaitGroup + p, _ := ants.NewPoolWithFunc(opt.Concurrency*5, func(i interface{}) { + instance := i.(CloudRunner) + + var options libs.Options + copier.Copy(&options, &opt) + instance.Opt = opt + instance.Provider.IsBackgroundCheck = true + instance.Provider.InitClient() + instance.Prepare() + utils.InforF("Deleting the instance: %v -- %v", instance.Provider.ProviderName, instance.PublicIP) + if err := instance.Provider.DeleteInstance(instance.InstanceID); err == nil { + utils.InforF("Instance deleted %s -- %s", color.HiYellowString(instance.PublicIP), color.HiYellowString(instance.InstanceID)) + instanceFile := instance.Opt.Env.InstancesFolder + "/" + instance.InstanceName + "-" + instance.PublicIP + ".json" + os.Remove(instanceFile) + } + + wg.Done() + }, ants.WithPreAlloc(true)) + defer p.Release() + + utils.InforF("Evaluating the status of %v cloud instances", color.HiMagentaString("%v", len(instances))) + for _, instance := range instances { + wg.Add(1) + _ = p.Invoke(instance) + } + wg.Wait() + +} diff --git a/distribute/cloud_runner.go b/distribute/cloud_runner.go new file mode 100644 index 0000000..da2ebcc --- /dev/null +++ b/distribute/cloud_runner.go @@ -0,0 +1,201 @@ +package distribute + +import ( + "fmt" + "os" + "strings" + "sync" + + "github.com/fatih/color" + "github.com/j3ssie/osmedeus/core" + "github.com/j3ssie/osmedeus/libs" + "github.com/j3ssie/osmedeus/provider" + "github.com/j3ssie/osmedeus/utils" + "github.com/panjf2000/ants" +) + +type CloudRunner struct { + Opt libs.Options `json:"-"` + Provider provider.Provider + + // for storing in local DB + TaskName string `json:"task_name"` + TaskType string `json:"task_type"` + Input string `json:"input"` + RawCommand string `json:"raw_command"` + InstanceFile string `json:"instance_file"` + + // core entry point + PublicIP string `json:"public_ip"` + DestInstance string `json:"dest_instance"` + SshPublicKey string `json:"ssh_public_key"` + SshPrivateKey string + SSHUser string + BasePath string + + InstanceID string `json:"instance_id"` + InstanceName string `json:"instance_name"` + IsError bool `json:"is_error"` + + Target map[string]string `json:"-"` + Runner core.Runner `json:"-"` +} + +// InitCloud init cloud runner obj +func InitCloud(options libs.Options, targets []string) { + // init clouds object queue + cloudConfigs := GetClouds(options) + if options.Cloud.CheckingLimit { + return + } + + // really start doing something + var wg sync.WaitGroup + p, _ := ants.NewPoolWithFunc(options.Concurrency, func(i interface{}) { + target := i.(string) + if strings.TrimSpace(target) == "" { + wg.Done() + return + } + + for { + var selectedCloud CloudRunner + var isDone bool + utils.DebugF("Select available accounts") + for _, cloudInfo := range cloudConfigs { + // don't run it again for next cloud account + if isDone { + break + } + if cloudInfo.Provider.Available { + selectedCloud = cloudInfo + } + + target = PrepareTarget(target, options) + if !options.Cloud.OnlyCreateDroplet { + utils.InforF("Initiating the scanning process of %v", color.HiMagentaString(utils.CleanPath(target))) + } + + // really start to run scan here + err := selectedCloud.Scan(target) + if err != nil { + utils.ErrorF("error start scan %s", color.HiCyanString(target)) + if options.Cloud.NoDelete { + continue + } + if ok := selectedCloud.Provider.DeleteInstance(selectedCloud.InstanceID); ok == nil { + selectedCloud.DeleteInstanceConfig() + } + continue + } + + if !options.Cloud.OnlyCreateDroplet { + utils.InforF("Completed scanning %v", color.HiCyanString(utils.CleanPath(target))) + utils.InforF("--------------------------------") + } + isDone = true + } + break + } + wg.Done() + + }, ants.WithPreAlloc(true)) + defer p.Release() + + for _, target := range targets { + wg.Add(1) + _ = p.Invoke(target) + } + wg.Wait() + +} + +// GetClouds prepare clouds object in config file +func GetClouds(options libs.Options) []CloudRunner { + var cloudInfos []CloudRunner + + // parse config from cloud/provider.yaml file + providerConfigs, err := provider.ParseProvider(options.CloudConfigFile) + if err != nil { + return cloudInfos + } + + options.Cloud.BuildRepo = providerConfigs.Builder.BuildRepo + options.Cloud.SecretKey = utils.NormalizePath(providerConfigs.Builder.SecretKey) + options.Cloud.PublicKey = utils.NormalizePath(providerConfigs.Builder.PublicKey) + + // we only get provider info from config file but replace token with --token + if options.Cloud.IgnoreConfigFile || options.Cloud.Token != "" { + var providerConfig provider.ConfigProvider + if len(providerConfigs.Clouds) > 0 { + utils.InforF("Ignore config file from: %v", options.CloudConfigFile) + providerConfig = providerConfigs.Clouds[0] + } + providerConfig.Token = options.Cloud.Token + cloudInfo := SetupProvider(options, providerConfig) + cloudInfos = append(cloudInfos, cloudInfo) + return cloudInfos + } + + for _, providerConfig := range providerConfigs.Clouds { + cloudInfo := SetupProvider(options, providerConfig) + if len(cloudInfo.Provider.Token) > 6 { + cloudInfo.Provider.RedactedToken = cloudInfo.Provider.Token[:5] + "***" + cloudInfo.Provider.Token[len(cloudInfo.Provider.Token)-5:] + } + cloudInfos = append(cloudInfos, cloudInfo) + } + + utils.InforF("Number of cloud providers ready in queue: %v", color.HiMagentaString("%v", len(cloudInfos))) + return cloudInfos +} + +// SetupProvider setup new provider +func SetupProvider(opt libs.Options, providerConfig provider.ConfigProvider) CloudRunner { + var cloudRunner CloudRunner + cloudRunner.Opt = opt + cloudRunner.Prepare() + + providerCloud, err := provider.InitProviderWithConfig(opt, providerConfig) + if err != nil { + return cloudRunner + } + cloudRunner.Provider = providerCloud + + cloudRunner.SSHUser = cloudRunner.Provider.SSHUser + if cloudRunner.SSHUser == "" { + cloudRunner.SSHUser = "root" + } + + if cloudRunner.SSHUser != "root" { + cloudRunner.BasePath = fmt.Sprintf("/home/%s", cloudRunner.SSHUser) + } + + if opt.Cloud.IgnoreSetup { + return cloudRunner + } + + // check if snapshot is okay or not + if !cloudRunner.Provider.SnapshotFound || opt.Cloud.ReBuildBaseImage { + err = cloudRunner.Provider.BuildImage() + if err != nil { + utils.ErrorF("error build snapshot at %v", cloudRunner.Provider.ProviderConfig.BuildFile) + return cloudRunner + } + } + + return cloudRunner +} + +// Prepare some variables +func (c *CloudRunner) Prepare() { + c.SshPrivateKey = c.Opt.Cloud.SecretKey + c.SshPublicKey = c.Opt.Cloud.PublicKey + c.SSHUser = "root" + c.BasePath = "/root" + + // make sure the permission of private key is right + os.Chmod(utils.NormalizePath(c.SshPrivateKey), 0600) + + // parse blank target to get env + c.Target = core.ParseInput("example.com", c.Opt) +} diff --git a/distribute/command.go b/distribute/command.go new file mode 100644 index 0000000..e2f3ecf --- /dev/null +++ b/distribute/command.go @@ -0,0 +1,175 @@ +package distribute + +import ( + "strings" + "time" + + "github.com/j3ssie/osmedeus/core" + "github.com/j3ssie/osmedeus/utils" +) + +func (c *CloudRunner) CloudMoreParams() { + utils.DebugF("Parsed more params") + + // take from -m flag + if c.Opt.Cloud.Module != "" { + module := core.DirectSelectModule(c.Opt, c.Opt.Cloud.Module) + parsedModule, err := core.ParseModules(module) + if err != nil || parsedModule.Name == "" { + return + } + + for _, param := range parsedModule.Params { + for k, v := range param { + v = core.ResolveData(v, c.Target) + if strings.HasPrefix(v, "~/") { + v = utils.NormalizePath(v) + } + c.Target[k] = v + } + } + + return + } + + flows := core.SelectFlow(c.Opt.Cloud.Flow, c.Opt) + if len(flows) == 0 { + return + } + + parsedFlow, err := core.ParseFlow(flows[0]) + if err != nil { + return + } + + for _, param := range parsedFlow.Params { + for k, v := range param { + v = core.ResolveData(v, c.Target) + if strings.HasPrefix(v, "~/") { + v = utils.NormalizePath(v) + } + c.Target[k] = v + } + } + +} + +func (c *CloudRunner) CreateUIReport() { + if c.Opt.NoDB { + return + } + + // take from -m flag + if c.Opt.Cloud.Module != "" { + module := core.DirectSelectModule(c.Opt, c.Opt.Cloud.Module) + parsedModule, err := core.ParseModules(module) + if err != nil || parsedModule.Name == "" { + return + } + + // create record on UI + c.Opt.Module = core.ResolveReports(parsedModule, c.Target) + // @TODO: add the report file to the runtime JSON file + // database.DBNewReports(parsedModule, &c.Runner.TargetObj) + return + } + + flows := core.SelectFlow(c.Opt.Cloud.Flow, c.Opt) + if len(flows) == 0 { + return + } + + parsedFlow, err := core.ParseFlow(flows[0]) + if err != nil { + return + } + + for _, rawModules := range parsedFlow.Routines { + // select module depend on it's flow + if rawModules.FlowFolder != "" { + parsedFlow.Type = rawModules.FlowFolder + } else { + parsedFlow.Type = parsedFlow.DefaultType + } + + modules := core.SelectModules(rawModules.Modules, c.Opt) + + for _, module := range modules { + parsedModule, err := core.ParseModules(module) + if err != nil || parsedModule.Desc == "" { + continue + } + // create record on UI + c.Opt.Module = core.ResolveReports(parsedModule, c.Target) + // @TODO: add the report runtime JSON file + // database.DBNewReports(c.Opt.Module, &c.Runner.TargetObj) + } + } +} + +func (c *CloudRunner) RetryCommandWithExpectString(cmd string, expectString string, timeoutRaw ...string) string { + timeout := "300s" + if len(timeoutRaw) > 0 { + timeout = timeoutRaw[0] + } + var out string + + utils.DebugF("Retry command: %s", cmd) + for i := 0; i < c.Opt.Cloud.Retry; i++ { + if timeout == "000" { + out, _ = utils.RunOSCommand(cmd) + } else { + out = utils.RunCmdWithOutput(cmd, timeout) + } + + if !strings.Contains(out, expectString) { + utils.DebugF(out) + time.Sleep(time.Duration(60*(i+1)) * time.Second) + continue + } + return out + } + return out +} + +func (c *CloudRunner) RetryCommandWithExcludeString(cmd string, excludeString string, timeoutRaw ...string) string { + var out string + timeout := "300s" + if len(timeoutRaw) > 0 { + timeout = timeoutRaw[0] + } + + utils.DebugF("Retry command: %s", cmd) + for i := 0; i < c.Opt.Cloud.Retry; i++ { + if timeout == "000" { + out, _ = utils.RunOSCommand(cmd) + } else { + out = utils.RunCmdWithOutput(cmd, timeout) + } + + if strings.Contains(out, excludeString) { + utils.DebugF(out) + time.Sleep(time.Duration(60*(i+1)) * time.Second) + continue + } + return out + } + return out +} + +func (c *CloudRunner) RetryCommand(cmd string, timeoutRaw ...string) { + timeout := "300s" + if len(timeoutRaw) > 0 { + timeout = timeoutRaw[0] + } + utils.DebugF("Retry command: %s", cmd) + for i := 0; i < c.Opt.Cloud.Retry; i++ { + out, err := utils.RunCommandWithErr(cmd, timeout) + if err != nil { + utils.DebugF(out) + time.Sleep(time.Duration(60*(i+1)) * time.Second) + continue + } + break + } +} diff --git a/distribute/create.go b/distribute/create.go new file mode 100644 index 0000000..c88cd0d --- /dev/null +++ b/distribute/create.go @@ -0,0 +1,153 @@ +package distribute + +import ( + "fmt" + "os" + "path" + "strings" + + "github.com/fatih/color" + "github.com/j3ssie/osmedeus/core" + "github.com/j3ssie/osmedeus/libs" + "github.com/j3ssie/osmedeus/utils" + "github.com/jinzhu/copier" + jsoniter "github.com/json-iterator/go" +) + +func (c *CloudRunner) CreateInstance(target string) error { + c.Opt.Cloud.Input = target + c.Target = core.ParseInput(target, c.Opt) + + if c.Opt.EnableFormatInput { + c.Opt.Cloud.Input = c.Target["Target"] + } + if c.Opt.Cloud.Workspace == "" { + c.Opt.Cloud.Workspace = utils.CleanPath(c.Target["Target"]) + } + + if c.Opt.Cloud.Workspace != "" { + c.Target["Workspace"] = c.Opt.Cloud.Workspace + // disable changing workspace name in huntersuite to keep track the scanID + if c.Opt.Cloud.EnableChunk { + if strings.Contains(target, "-chunk-") { + index := strings.Split(target, "-chunk-")[1] + c.Target["WorkspaceChunk"] = path.Base(c.Opt.Cloud.Workspace) + "-" + index + utils.DebugF("Changing workspace name: %v", c.Target["WorkspaceChunk"]) + } + } + utils.DebugF(`c.Target["Workspace"] -- %v`, c.Target["Workspace"]) + } + + // run-flow-example.com + InstancePrefix := fmt.Sprintf("run") + if c.Opt.Cloud.Flow != "" { + InstancePrefix = fmt.Sprintf("run-%s", utils.CleanPath(c.Opt.Cloud.Flow)) + } + if c.Opt.Cloud.Module != "" { + InstancePrefix = fmt.Sprintf("run-%s", utils.CleanPath(c.Opt.Cloud.Module)) + } + + // run-flow-example.com-1 + c.InstanceName = fmt.Sprintf("%s-%s", InstancePrefix, strings.TrimSpace(c.Target["Workspace"])) + if c.Opt.Cloud.EnableChunk { + c.InstanceName = fmt.Sprintf("%s-%s", InstancePrefix, strings.TrimSpace(c.Target["WorkspaceChunk"])) + } + + // make sure the droplet name is unique + c.InstanceName = c.InstanceName + "-" + utils.RandomString(4) + // clean up the instance name first + if strings.Contains(c.InstanceName, "_") { + c.InstanceName = strings.ReplaceAll(c.InstanceName, "_", "-") + } + + // force changing instance name from cli + if c.Opt.Cloud.InstanceName != "" { + c.InstanceName = c.Opt.Cloud.InstanceName + } + + // quick check for instance name + switch c.Provider.ProviderName { + case "ln", "line", "linode": + if len(c.InstanceName) > 32 { + c.InstanceName = strings.Trim(strings.Trim(strings.Trim(c.InstanceName[:20], "-"), "."), "_") + "-" + utils.RandomString(9) + } + } + + /* Really start to run command to create instance here */ + err := c.Provider.CreateInstance(c.InstanceName) + + // fix the naming in linode might need another check for length + if err != nil { + // check if account reach limit first + if strings.Contains(err.Error(), "Account Limit reached") { + utils.ErrorF("Account %v reach limit instance", c.Provider.RedactedToken) + return fmt.Errorf("error creating instance") + } + + if strings.Contains(err.Error(), "valid hostname characters are allowed") || strings.Contains(err.Error(), "[400] [label]") { + c.InstanceName = fmt.Sprintf("runr-%s-%s", utils.GetTS(), utils.RandomString(8)) + } + err = c.Provider.CreateInstance(c.InstanceName) + if err != nil { + return fmt.Errorf("error creating instance") + } + } + + if err != nil { + return fmt.Errorf("error creating instance: %v", c.InstanceName) + } + + c.PublicIP = c.Provider.CreatedInstance.IPAddress + + c.InstanceID = c.Provider.CreatedInstance.InstanceID + c.DestInstance = fmt.Sprintf("%s@%s", c.SSHUser, c.PublicIP) + utils.InforF("Instance created: %s -- %s -- %v", color.HiCyanString(c.InstanceName), color.HiCyanString(c.InstanceID), color.HiCyanString(c.PublicIP)) + c.Target["CIP"] = c.PublicIP + c.Target["RemoteIP"] = c.PublicIP + + // create the JSON file for the instance + c.WriteInstanceConfig() + return nil + +} + +func (c *CloudRunner) DeleteInstanceConfig() { + utils.DebugF("Deleting instance config %v", color.HiCyanString(c.InstanceFile)) + os.Remove(c.InstanceFile) +} + +func (c *CloudRunner) WriteInstanceConfig() error { + instanceFile := c.Opt.Env.InstancesFolder + "/" + c.InstanceName + "-" + c.PublicIP + ".json" + c.InstanceFile = instanceFile + utils.DebugF("Writing instance config to %v", color.HiCyanString(instanceFile)) + + var instanceObj CloudRunner + copier.Copy(&instanceObj, &c) + instanceObj.Runner = core.Runner{} + instanceObj.Opt = libs.Options{} + + if data, err := jsoniter.MarshalToString(instanceObj); err == nil { + utils.WriteToFile(instanceFile, data) + } + + return nil +} + +func (c *CloudRunner) ErrorWriteInstanceConfig() error { + instanceFile := c.Opt.Env.InstancesFolder + "/" + c.InstanceName + "-" + c.PublicIP + ".json" + c.InstanceFile = instanceFile + utils.DebugF("Writing instance config to %v", color.HiCyanString(instanceFile)) + + var instanceObj CloudRunner + copier.Copy(&instanceObj, &c) + + instanceObj.IsError = true + instanceObj.Runner = core.Runner{} + instanceObj.Opt = libs.Options{} + + if data, err := jsoniter.MarshalToString(instanceObj); err == nil { + utils.WriteToFile(instanceFile, data) + } + + return nil +} diff --git a/distribute/db_cloud.go b/distribute/db_cloud.go new file mode 100644 index 0000000..43afa24 --- /dev/null +++ b/distribute/db_cloud.go @@ -0,0 +1,35 @@ +package distribute + +import ( + "path" + "time" + + "github.com/fatih/color" + "github.com/j3ssie/osmedeus/database" + "github.com/j3ssie/osmedeus/utils" + jsoniter "github.com/json-iterator/go" +) + +func (c *CloudRunner) DBNewTarget() { + targetFolder := path.Join(c.Opt.Env.WorkspacesFolder, c.Target["Workspace"]) + utils.MakeDir(targetFolder) + + c.Runner.ScanObj = database.Scan{ + IsCloud: true, + InputName: c.Target["Workspace"], + TaskType: "distributed", + } + + c.Runner.ScanObj.CreatedAt = time.Now() + c.DBRuntimeUpdate() +} + +func (c *CloudRunner) DBRuntimeUpdate() { + runtimeFile := path.Join(c.Opt.Env.WorkspacesFolder, c.Target["Workspace"], "runtime") + c.Runner.ScanObj.UpdatedAt = time.Now() + + if runtimeData, err := jsoniter.MarshalToString(c.Runner.ScanObj); err == nil { + utils.InforF("Updating runtime file: %s", color.HiCyanString(runtimeFile)) + utils.WriteToFile(runtimeFile, runtimeData) + } +} diff --git a/distribute/health.go b/distribute/health.go new file mode 100644 index 0000000..6994de6 --- /dev/null +++ b/distribute/health.go @@ -0,0 +1,159 @@ +package distribute + +import ( + "fmt" + "os" + "path" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/fatih/color" + "github.com/j3ssie/osmedeus/libs" + "github.com/j3ssie/osmedeus/utils" + "github.com/jinzhu/copier" + jsoniter "github.com/json-iterator/go" + "github.com/panjf2000/ants" +) + +func GetAllInstances(opt libs.Options) (instances []CloudRunner) { + utils.InforF("Getting all instances from %s", opt.Env.InstancesFolder) + + // Read all entries in the directory + entries, err := os.ReadDir(opt.Env.InstancesFolder) + if err != nil { + utils.ErrorF("error reading instances folder: %v", err) + return instances + } + + for _, entry := range entries { + instanceFile := filepath.Join(opt.Env.InstancesFolder, entry.Name()) + + if !utils.FileExists(instanceFile) { + continue + } + + instance := CloudRunner{} + runtimeContent := utils.GetFileContent(instanceFile) + err := jsoniter.UnmarshalFromString(runtimeContent, &instance) + if err != nil { + utils.ErrorF("error marshal data: %v", err) + } + instances = append(instances, instance) + } + + return instances +} + +func CheckingCloudInstance(opt libs.Options) { + if opt.Cloud.Retry == 0 { + opt.Cloud.Retry = 8 + } + + // select all cloud instance + instances := GetAllInstances(opt) + if len(instances) == 0 { + utils.WarnF("no active cloud instance running") + return + } + + var wg sync.WaitGroup + p, _ := ants.NewPoolWithFunc(opt.Concurrency*5, func(i interface{}) { + // really start to scan + instance := i.(CloudRunner) + + var options libs.Options + copier.Copy(&options, &opt) + instance.Opt = opt + instance.Provider.IsBackgroundCheck = true + instance.Provider.InitClient() + instance.Prepare() + instance.HealthCheck() + wg.Done() + }, ants.WithPreAlloc(true)) + defer p.Release() + + utils.InforF("Evaluating the status of %v cloud instances", color.HiMagentaString("%v", len(instances))) + for _, instance := range instances { + wg.Add(1) + _ = p.Invoke(instance) + } + wg.Wait() + +} + +func (c *CloudRunner) HealthCheck() bool { + utils.InforF("[%v] Checking the instance: %v -- %v", color.HiYellowString(c.Provider.ProviderName), color.HiCyanString(c.PublicIP), color.HiCyanString(path.Base(c.InstanceName))) + c.Opt.Cloud.Input = c.Input + + counter := 0 + for i := 0; i < c.Opt.Cloud.Retry; i++ { + // sync in result first + c.SyncResult() + if !c.IsRunning() || c.IsPanic() { + counter += 1 + utils.DebugF("retry[%v]: An error has been detected at the cloud instance: %v -- %v", counter, c.Provider.ProviderName, c.PublicIP) + } + + if counter == c.Opt.Cloud.Retry-1 { + utils.ErrorF("An error has been detected at the cloud instance: %v -- %v", c.Provider.ProviderName, c.PublicIP) + err := c.Provider.DeleteInstance(c.InstanceID) + if err == nil { + utils.InforF("Instance deleted %s -- %s", color.HiYellowString(c.PublicIP), color.HiYellowString(c.InstanceID)) + instanceFile := c.Opt.Env.InstancesFolder + "/" + c.InstanceName + "-" + c.PublicIP + ".json" + os.Remove(instanceFile) + } + return false + } + time.Sleep(10 * time.Second) + } + + utils.InforF("[%v] Instance is still running well: %v -- %v", color.HiYellowString(c.Provider.ProviderName), color.HiCyanString(c.PublicIP), color.HiCyanString(path.Base(c.InstanceName))) + return true +} + +// IsRunning checking if cloud instance is running or not +func (c *CloudRunner) IsRunning() bool { + utils.DebugF("Checking running process at: %v", c.PublicIP) + cmd := fmt.Sprintf("%s utils ps --json", libs.BINARY) + + // ignore checking process if you're running custom command '--no-ps' + if c.Opt.Cloud.IgnoreProcess { + return true + } + + out, err := c.SSHExec(cmd) + if err == nil && strings.Contains(out, "pid") { + return true + } + + // retry checking process + for i := 0; i < c.Opt.Cloud.Retry; i += 2 { + out, err := c.SSHExec(cmd) + if err == nil && strings.Contains(out, "pid") { + return true + } + } + + utils.DebugF(out) + utils.ErrorF("no process running at %v", c.PublicIP) + return false +} + +// IsPanic checking if cloud instance has any panic error +func (c *CloudRunner) IsPanic() bool { + utils.DebugF("Checking panic error at: %v", c.PublicIP) + cmd := fmt.Sprintf("%s utils tmux logs -A -l 30", libs.BINARY) + out, err := c.SSHExec(cmd) + + if err == nil { + if strings.Contains(out, "out of memory") || strings.Contains(out, "runtime.(*") || strings.Contains(out, "[panic]") { + utils.DebugF(out) + utils.ErrorF("Fatal panic detected at: %s", c.PublicIP) + return true + } + } + + return false +} diff --git a/distribute/mics.go b/distribute/mics.go new file mode 100644 index 0000000..3222aaa --- /dev/null +++ b/distribute/mics.go @@ -0,0 +1,115 @@ +package distribute + +import ( + "fmt" + "path" + "strings" + + "github.com/j3ssie/osmedeus/libs" + "github.com/j3ssie/osmedeus/utils" + "github.com/spf13/cast" +) + +// CommandBuilder build core command from API +func CommandBuilder(options libs.Options) string { + binary := libs.BINARY + " --from-remote" + + if options.Debug { + binary += " --debug" + } + + if options.Update.EnableUpdate { + binary += " --update" + } + + if options.Threads > 0 { + binary += " --threads-hold " + cast.ToString(options.Threads) + } + + if options.ScanID != "" { + binary += fmt.Sprintf(" --sid %v ", options.ScanID) + } + + taskData := options.Cloud + var command string + var workspace, concurrency, timeout, params, workflow, plugin, extra string + if options.Cloud.Extra != "" { + extra = " " + options.Cloud.Extra + " " + } + + if options.Cloud.Flow != "" { + options.Cloud.Flow = utils.NormalizePath(options.Cloud.Flow) + workflow = fmt.Sprintf(" -f '%v' ", options.Cloud.Flow) + } + + if len(taskData.Params) > 0 { + for _, param := range taskData.Params { + params += fmt.Sprintf(" -p '%v'", param) + } + } + + if options.Cloud.Threads > 1 { + concurrency = fmt.Sprintf(" -c %d ", options.Cloud.Threads) + } + + // get workspace + if taskData.Workspace == "" { + taskData.Workspace = utils.CleanPath(taskData.Input) + } + if taskData.Workspace != "" { + workspace = fmt.Sprintf(" -w '%v'", strings.TrimSpace(taskData.Workspace)) + } + + if taskData.Module != "" { + taskData.Module = utils.NormalizePath(taskData.Module) + plugin = fmt.Sprintf(" -m '%v' ", taskData.Module) + } + + // override everything + if taskData.RawCommand != "" { + return taskData.RawCommand + } + + // use target as a file + if options.Cloud.TargetAsFile && utils.FileExists(taskData.Input) { + taskData.InputsFile = taskData.Input + } + + // mean general scan + if taskData.Module == "" { + command = fmt.Sprintf("%v scan %v -t %v %v%v%v%v%v", binary, workflow, taskData.Input, concurrency, timeout, workspace, params, extra) + if options.Cloud.TargetAsFile { + command = fmt.Sprintf("%v scan %v -T %v %v%v%v%v%v", binary, workflow, taskData.InputsFile, concurrency, timeout, workspace, params, extra) + } + command = strings.TrimSpace(command) + + return command + } + + command = fmt.Sprintf("%v scan %v -t %v %v%v%v%v%v", binary, plugin, taskData.Input, concurrency, timeout, workspace, params, extra) + if options.Cloud.TargetAsFile { + command = fmt.Sprintf("%v scan %v -T %v %v%v%v%v%v", binary, plugin, taskData.InputsFile, concurrency, timeout, workspace, params, extra) + } + command = strings.TrimSpace(command) + + return command +} + +// PrepareTarget change the target file destination +func PrepareTarget(target string, options libs.Options) string { + if options.Cloud.EnableChunk { + return target + } + + if !utils.FileExists(target) && !utils.FolderExists(target) { + utils.DebugF("target is not a file: %s", target) + return target + } + + utils.MakeDir(options.Cloud.TempTarget) + dest := path.Join(options.Cloud.TempTarget, path.Base(target)) + + utils.Copy(target, dest) + utils.InforF("Change target %s --> %s", target, dest) + return dest +} diff --git a/distribute/routine.go b/distribute/routine.go new file mode 100644 index 0000000..589b7e8 --- /dev/null +++ b/distribute/routine.go @@ -0,0 +1,94 @@ +package distribute + +import ( + "fmt" + "time" + + "github.com/j3ssie/osmedeus/core" + "github.com/j3ssie/osmedeus/libs" + "github.com/j3ssie/osmedeus/utils" +) + +func (c *CloudRunner) Scan(target string) error { + err := c.CreateInstance(target) + if err != nil { + utils.ErrorF("Error to create instance") + return err + } + + if c.Opt.Cloud.OnlyCreateDroplet { + utils.DebugF("Only create instance, skip the scan") + return nil + } + + // parse some parameters first + c.PrepareInput() + + // pre run before starting the scan + c.PreRunLocal() + + if c.Opt.Cloud.EnableSyncWorkflow { + err = c.CopyWorkflow() + if err != nil { + utils.ErrorF("Error to copy workflow to instance") + return err + } + } + + // copy target to droplet first + err = c.CopyTarget() + if err != nil { + utils.ErrorF("Error to copy input to instance") + return err + } + + // pre run commands + c.PreRunRemote() + + err = c.StartScan() + if err != nil { + utils.ErrorF("Error to run command on instance") + return err + } + + // check if done file created in instance or not + c.CheckingDone() + + if !c.Opt.Cloud.DisableLocalSync { + err = c.SyncResult() + // sync result to one more time if it's not done yet + if c.Opt.Cloud.NoDelete { + time.Sleep(100 * time.Second) + err = c.SyncResult() + } + + if err != nil { + utils.ErrorF("Error to sync result to instance") + return err + } + } + + // post run after scan done + c.PostRunLocal() + + if c.Opt.Cloud.CopyWorkspaceToGit { + utils.InforF("Coping workspace to git storages") + baseCmd := fmt.Sprintf("%s scan --nn -f sync -t {{.Workspace}}", libs.BINARY) + cmd := core.ResolveData(baseCmd, c.Target) + utils.RunCommandWithErr(cmd) + } + + if c.Opt.Cloud.NoDelete { + utils.DebugF("Skipping delete instance") + return nil + } + + err = c.Provider.DeleteInstance(c.InstanceID) + if err != nil { + utils.ErrorF("Error to delete instance") + return err + } + c.DeleteInstanceConfig() + + return nil +} diff --git a/distribute/scan.go b/distribute/scan.go new file mode 100644 index 0000000..7a38b88 --- /dev/null +++ b/distribute/scan.go @@ -0,0 +1,267 @@ +package distribute + +import ( + "fmt" + "path" + "strings" + "time" + + "github.com/fatih/color" + "github.com/j3ssie/osmedeus/core" + "github.com/j3ssie/osmedeus/libs" + "github.com/j3ssie/osmedeus/provider" + "github.com/j3ssie/osmedeus/utils" +) + +func (c *CloudRunner) PrepareInput() { + c.Opt.Scan.ROptions = c.Target + c.Opt.Scan.Flow = "cloud-distributed" + //database.NewScan(c.Opt, "cli") + c.Input = c.Target["Target"] + runner, err := core.InitRunner(c.Input, c.Opt) + if err == nil { + c.Runner = runner + c.Runner.RunnerType = "cloud" + } + + // for creating local DB record + if c.Opt.Cloud.Flow != "" { + c.TaskName = c.Opt.Cloud.Flow + c.TaskType = "flow" + } else { + c.TaskName = c.Opt.Cloud.Flow + c.TaskType = "module" + } + c.CloudMoreParams() + // more params from -p flag + if len(c.Opt.Cloud.Params) > 0 { + params := core.ParseParams(c.Opt.Cloud.Params) + if len(params) > 0 { + for k, v := range params { + v = core.ResolveData(v, c.Target) + c.Target[k] = v + } + } + } + +} + +func (c *CloudRunner) StartScan() error { + c.DBNewTarget() + // c.DBNewScanLocal() + // c.DBNewCloudInstance() + + err := c.RunScan() + if err != nil { + return fmt.Errorf("error to start the scan") + } + + // utils.DebugF("Create UI report for %s: %s", c.DestInstance, color.HiCyanString(c.Opt.Cloud.RawCommand)) + // c.Runner.DBDoneScan() + return nil +} + +func (c *CloudRunner) RunScan() error { + // -f mean run in a background + // ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i root@IP -f + // -t mean run and wait + // ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i root@IP -t + if c.Opt.Cloud.RawCommand != "" { + c.Opt.Cloud.RawCommand = core.ResolveData(c.Opt.Cloud.RawCommand, c.Target) + } else { + c.Opt.Cloud.RawCommand = CommandBuilder(c.Opt) + } + c.Opt.Cloud.RawCommand = core.ResolveData(c.Opt.Cloud.RawCommand, c.Target) + c.RawCommand = c.Opt.Cloud.RawCommand + + // init tmux session + out, err := c.SSHExec("tmux new-session -d -t main") + + // boot the instance again if it still didn't up + if err != nil && strings.Contains(err.Error(), "time out") { + time.Sleep(60 * time.Second) + c.Provider.Action(provider.BootInstance, c.InstanceID) + out, err = c.SSHExec("tmux new-session -d -t main") + } + + // really run main command + tcmd := fmt.Sprintf(`"%s"`, c.Opt.Cloud.RawCommand) + _, err = c.SSHExec(fmt.Sprintf(`tmux send-keys %s ENTER`, tcmd)) + + // still error then it must be something wrong + if err != nil { + utils.ErrorF("An error occurred with %v", color.HiYellowString(c.DestInstance)) + utils.ErrorF("error log: %v", out) + return fmt.Errorf("error running command on %v", color.HiYellowString(c.DestInstance)) + } + utils.InforF("Start to run the scan %v with command %v", color.HiYellowString(c.DestInstance), color.HiCyanString(c.Opt.Cloud.RawCommand)) + + // wait a bit for process really start + time.Sleep(60 * time.Second) + if !c.IsRunning() { + return fmt.Errorf("Failed to initiate the scan on %v", color.HiYellowString(c.DestInstance)) + } + + c.WriteInstanceConfig() + return nil +} + +func (c *CloudRunner) CheckingDone() error { + if c.Opt.Cloud.NoDelete { + time.Sleep(60 * time.Second) + return nil + } + utils.InforF("Checking scan process at: %s", color.HiBlueString(c.PublicIP)) + + // dest := fmt.Sprintf("%s/.%s/workspaces/%s/done", c.BasePath, libs.BINARY, c.Target["Workspace"]) + // @NOTE: this is new workspaces folder + dest := fmt.Sprintf("%s/workspaces-%s/%s/done", c.BasePath, libs.BINARY, c.Target["Workspace"]) + + cmd := fmt.Sprintf("file %s", dest) + out, _ := c.SSHExec(cmd) + + if strings.Contains(out, "ASCII text") || strings.Contains(out, "JSON data") { + return nil + } + + waitTime := utils.CalcTimeout(c.Opt.Cloud.ClearTime) + counter := 1 + for { + time.Sleep(time.Duration(waitTime) * time.Second) + out, _ = c.SSHExec(cmd) + if strings.Contains(out, "ASCII text") || strings.Contains(out, "JSON data") { + utils.InforF("The scan is done at: %s", color.HiBlueString(c.PublicIP)) + return nil + } + + if !c.IsRunning() { + return fmt.Errorf("no process running at %v", c.PublicIP) + } + + // check if we have panic or not + if c.IsPanic() { + return fmt.Errorf("panic detected at %v", c.PublicIP) + } + + if counter%50 == 0 { + c.SyncResult() + } + counter++ + } +} + +// below code is experimental part + +func (c *CloudRunner) SyncResult() error { + target := c.Opt.Cloud.Input + if !c.Provider.IsBackgroundCheck { + utils.InforF("Sync back the data of taget %v from %v", color.HiCyanString(target), color.HiYellowString(c.DestInstance)) + } + + if c.Opt.Cloud.LocalSyncFolder == "" { + c.Opt.Cloud.LocalSyncFolder = fmt.Sprintf("%s/workspaces-%s/", c.BasePath, libs.BINARY) + } + + // on vps machine + src := c.Opt.Cloud.LocalSyncFolder + + // on local + dest := path.Join(c.Opt.Env.WorkspacesFolder, c.Opt.Cloud.BaseWorkspace) + utils.MakeDir(dest) + + cmd := fmt.Sprintf("rsync -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s' -avzr --progress %s:%s %s", c.SshPrivateKey, c.DestInstance, src, dest) + + c.RetryCommandWithExpectString(cmd, `bytes/sec`) + if !utils.FolderExists(dest) { + utils.ErrorF("error sync result back from: %v to %v", c.DestInstance, dest) + } + + return nil +} + +func (c *CloudRunner) CopyTarget() error { + target := c.Opt.Cloud.Input + utils.DebugF("Sync input of %s to %s", target, c.DestInstance) + + dest := c.Target["Target"] + if !utils.FileExists(dest) && !utils.FolderExists(dest) { + utils.DebugF("target is not a file: %s", dest) + return nil + } + + if c.Opt.Cloud.EnableChunk { + dest = c.Opt.Cloud.ChunkInputs + } + + c.SSHExec(fmt.Sprintf("mkdir -p %s", path.Dir(dest))) + cmd := fmt.Sprintf("rsync -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s' -avzr --progress %s %s:%s", c.SshPrivateKey, dest, c.DestInstance, dest) + c.RetryCommandWithExpectString(cmd, `bytes/sec`) + return nil +} + +func (c *CloudRunner) CopyWorkflow() error { + utils.DebugF("Sync workflow of %s to %s", c.Opt.Env.WorkFlowsFolder, c.DestInstance) + destWorkflow := fmt.Sprintf("%v/osmedeus-base/", c.BasePath) + if c.Opt.Cloud.RemoteWorkflowFolder != "" { + destWorkflow = c.Opt.Cloud.RemoteWorkflowFolder + } + + // c.SSHExec(fmt.Sprintf("rm -rf %s && mkdir -p %s", destWorkflow, destWorkflow)) + cmd := fmt.Sprintf("rsync -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s' -avzr --progress %s %s:%s", c.SshPrivateKey, c.Opt.Env.WorkFlowsFolder, c.DestInstance, destWorkflow) + c.RetryCommandWithExpectString(cmd, `bytes/sec`) + return nil +} + +func (c *CloudRunner) PreRunRemote() error { + if len(c.Opt.Cloud.RemotePreRun) <= 0 { + return nil + } + utils.InforF("Run remote command on: %s", c.PublicIP) + + // really start to run pre commands + for _, rcmd := range c.Opt.Cloud.RemotePreRun { + cmd := core.ResolveData(rcmd, c.Target) + utils.InforF("Run pre command on %s: %s", c.PublicIP, cmd) + c.SSHExec(cmd) + } + return nil +} + +func (c *CloudRunner) PreRunLocal() error { + if len(c.Opt.Cloud.LocalPreRun) <= 0 { + return nil + } + c.Opt.Scan.ROptions = c.Target + utils.InforF("Start %v", color.HiCyanString("PreRunLocal")) + + // really start to run pre commands + for _, script := range c.Opt.Cloud.LocalPreRun { + script = core.ResolveData(script, c.Target) + c.Runner.RunScript(script) + } + return nil +} + +func (c *CloudRunner) PostRunLocal() error { + c.Opt.Scan.ROptions = c.Target + + if len(c.Opt.Cloud.LocalSteps) > 0 { + // for running local steps + utils.DebugF("Running local steps") + for _, step := range c.Opt.Cloud.LocalSteps { + c.Runner.RunStep(step) + } + } + + if len(c.Opt.Cloud.LocalPostRun) <= 0 { + return nil + } + + utils.InforF("Start %v", color.HiCyanString("PostRunLocal")) + // really start to run pre commands + for _, script := range c.Opt.Cloud.LocalPostRun { + script = core.ResolveData(script, c.Target) + c.Runner.RunScript(script) + } + return nil +} diff --git a/distribute/ssh.go b/distribute/ssh.go new file mode 100644 index 0000000..f7e2c44 --- /dev/null +++ b/distribute/ssh.go @@ -0,0 +1,430 @@ +package distribute + +import ( + "bytes" + "errors" + "fmt" + "io" + "net" + "os" + "strings" + "time" + + "github.com/fatih/color" + "github.com/j3ssie/osmedeus/utils" + + "golang.org/x/crypto/ssh" +) + +func (c *CloudRunner) SSHExec(command string) (string, error) { + client, err := c.InitSSHClient() + if err != nil { + utils.ErrorF("Failed to initialize SSH connection to %v", color.HiYellowString(c.PublicIP)) + return "", err + } + defer client.Close() + + utils.DebugF("Running command on %s: %s", color.HiBlueString(c.PublicIP), color.HiGreenString(command)) + out, err := client.Cmd(command).Output() + if err != nil { + utils.ErrorF("err run command: %v", err) + return "", err + } + + return string(out), nil +} + +func (c *CloudRunner) InitSSHClient() (*Client, error) { + host := fmt.Sprintf("%s:%s", c.PublicIP, "22") + dest := color.HiCyanString("%s@%s", c.SSHUser, host) + utils.DebugF("Connecting to %v with key %v", dest, c.SshPrivateKey) + client, err := DialWithKeyString(host, c.SSHUser, c.Opt.Cloud.SecretKeyContent) + if err != nil { + utils.ErrorF("err connect to %v -- %v", host, err) + for i := 0; i < c.Opt.Cloud.Retry; i++ { + client, err = DialWithKeyString(host, c.SSHUser, c.Opt.Cloud.SecretKeyContent) + if err != nil { + time.Sleep(time.Duration(30*(i+1)) * time.Second) + continue + } + break + } + if err != nil { + return nil, err + } + } + return client, nil +} + +/* Start of SSH Lib */ +// literally copy from this: https://github.com/helloyi/go-sshclient + +type remoteScriptType byte +type remoteShellType byte + +const ( + cmdLine remoteScriptType = iota + rawScript + scriptFile + + interactiveShell remoteShellType = iota + nonInteractiveShell +) + +type Client struct { + client *ssh.Client +} + +// DialWithPasswd starts a client connection to the given SSH server with passwd authmethod. +func DialWithPasswd(addr, user, passwd string) (*Client, error) { + config := &ssh.ClientConfig{ + User: user, + Auth: []ssh.AuthMethod{ + ssh.Password(passwd), + }, + HostKeyCallback: ssh.HostKeyCallback(func(hostname string, remote net.Addr, key ssh.PublicKey) error { return nil }), + } + + return Dial("tcp", addr, config) +} + +// DialWithKeyString starts a client connection to the given SSH server with key authmethod. +func DialWithKeyString(addr, user, keyContent string) (*Client, error) { + signer, err := ssh.ParsePrivateKey([]byte(strings.TrimSpace(keyContent))) + if err != nil { + return nil, err + } + + config := &ssh.ClientConfig{ + User: user, + Auth: []ssh.AuthMethod{ + ssh.PublicKeys(signer), + }, + HostKeyCallback: ssh.HostKeyCallback(func(hostname string, remote net.Addr, key ssh.PublicKey) error { return nil }), + } + + return Dial("tcp", addr, config) +} + +// DialWithKey starts a client connection to the given SSH server with key authmethod. +func DialWithKey(addr, user, keyfile string) (*Client, error) { + key, err := os.ReadFile(keyfile) + if err != nil { + return nil, err + } + + signer, err := ssh.ParsePrivateKey(key) + if err != nil { + return nil, err + } + + config := &ssh.ClientConfig{ + User: user, + Auth: []ssh.AuthMethod{ + ssh.PublicKeys(signer), + }, + HostKeyCallback: ssh.HostKeyCallback(func(hostname string, remote net.Addr, key ssh.PublicKey) error { return nil }), + } + + return Dial("tcp", addr, config) +} + +// DialWithKeyWithPassphrase same as DialWithKey but with a passphrase to decrypt the private key +func DialWithKeyWithPassphrase(addr, user, keyfile string, passphrase string) (*Client, error) { + key, err := os.ReadFile(keyfile) + if err != nil { + return nil, err + } + + signer, err := ssh.ParsePrivateKeyWithPassphrase(key, []byte(passphrase)) + if err != nil { + return nil, err + } + + config := &ssh.ClientConfig{ + User: user, + Auth: []ssh.AuthMethod{ + ssh.PublicKeys(signer), + }, + HostKeyCallback: ssh.HostKeyCallback(func(hostname string, remote net.Addr, key ssh.PublicKey) error { return nil }), + } + + return Dial("tcp", addr, config) +} + +// Dial starts a client connection to the given SSH server. +// This is wrap the ssh.Dial +func Dial(network, addr string, config *ssh.ClientConfig) (*Client, error) { + client, err := ssh.Dial(network, addr, config) + if err != nil { + return nil, err + } + return &Client{ + client: client, + }, nil +} + +func (c *Client) Close() error { + return c.client.Close() +} + +// Cmd create a command on client +func (c *Client) Cmd(cmd string) *remoteScript { + return &remoteScript{ + _type: cmdLine, + client: c.client, + script: bytes.NewBufferString(cmd + "\n"), + } +} + +// Script +func (c *Client) Script(script string) *remoteScript { + return &remoteScript{ + _type: rawScript, + client: c.client, + script: bytes.NewBufferString(script + "\n"), + } +} + +// ScriptFile +func (c *Client) ScriptFile(fname string) *remoteScript { + return &remoteScript{ + _type: scriptFile, + client: c.client, + scriptFile: fname, + } +} + +type remoteScript struct { + client *ssh.Client + _type remoteScriptType + script *bytes.Buffer + scriptFile string + err error + + stdout io.Writer + stderr io.Writer +} + +// Run +func (rs *remoteScript) Run() error { + if rs.err != nil { + fmt.Println(rs.err) + return rs.err + } + + if rs._type == cmdLine { + return rs.runCmds() + } else if rs._type == rawScript { + return rs.runScript() + } else if rs._type == scriptFile { + return rs.runScriptFile() + } else { + return errors.New("Not supported remoteScript type") + } +} + +func (rs *remoteScript) Output() ([]byte, error) { + if rs.stdout != nil { + return nil, errors.New("Stdout already set") + } + var out bytes.Buffer + rs.stdout = &out + err := rs.Run() + return out.Bytes(), err +} + +func (rs *remoteScript) SmartOutput() ([]byte, error) { + if rs.stdout != nil { + return nil, errors.New("Stdout already set") + } + if rs.stderr != nil { + return nil, errors.New("Stderr already set") + } + + var ( + stdout bytes.Buffer + stderr bytes.Buffer + ) + rs.stdout = &stdout + rs.stderr = &stderr + err := rs.Run() + if err != nil { + return stderr.Bytes(), err + } + return stdout.Bytes(), err +} + +func (rs *remoteScript) Cmd(cmd string) *remoteScript { + _, err := rs.script.WriteString(cmd + "\n") + if err != nil { + rs.err = err + } + return rs +} + +func (rs *remoteScript) SetStdio(stdout, stderr io.Writer) *remoteScript { + rs.stdout = stdout + rs.stderr = stderr + return rs +} + +func (rs *remoteScript) runCmd(cmd string) error { + session, err := rs.client.NewSession() + if err != nil { + return err + } + defer session.Close() + + session.Stdout = rs.stdout + session.Stderr = rs.stderr + + if err := session.Run(cmd); err != nil { + return err + } + return nil +} + +func (rs *remoteScript) runCmds() error { + for { + statment, err := rs.script.ReadString('\n') + if err == io.EOF { + break + } + if err != nil { + return err + } + + if err := rs.runCmd(statment); err != nil { + return err + } + } + + return nil +} + +func (rs *remoteScript) runScript() error { + session, err := rs.client.NewSession() + if err != nil { + return err + } + + session.Stdin = rs.script + session.Stdout = rs.stdout + session.Stderr = rs.stderr + + if err := session.Shell(); err != nil { + return err + } + if err := session.Wait(); err != nil { + return err + } + + return nil +} + +func (rs *remoteScript) runScriptFile() error { + var buffer bytes.Buffer + file, err := os.Open(rs.scriptFile) + if err != nil { + return err + } + defer file.Close() + _, err = io.Copy(&buffer, file) + if err != nil { + return err + } + + rs.script = &buffer + return rs.runScript() +} + +type remoteShell struct { + client *ssh.Client + requestPty bool + terminalConfig *TerminalConfig + + stdin io.Reader + stdout io.Writer + stderr io.Writer +} + +type TerminalConfig struct { + Term string + Height int + Weight int + Modes ssh.TerminalModes +} + +// Terminal create a interactive shell on client. +func (c *Client) Terminal(config *TerminalConfig) *remoteShell { + return &remoteShell{ + client: c.client, + terminalConfig: config, + requestPty: true, + } +} + +// Shell create a noninteractive shell on client. +func (c *Client) Shell() *remoteShell { + return &remoteShell{ + client: c.client, + requestPty: false, + } +} + +func (rs *remoteShell) SetStdio(stdin io.Reader, stdout, stderr io.Writer) *remoteShell { + rs.stdin = stdin + rs.stdout = stdout + rs.stderr = stderr + return rs +} + +// Start start a remote shell on client +func (rs *remoteShell) Start() error { + session, err := rs.client.NewSession() + if err != nil { + return err + } + defer session.Close() + + if rs.stdin == nil { + session.Stdin = os.Stdin + } else { + session.Stdin = rs.stdin + } + if rs.stdout == nil { + session.Stdout = os.Stdout + } else { + session.Stdout = rs.stdout + } + if rs.stderr == nil { + session.Stderr = os.Stderr + } else { + session.Stderr = rs.stderr + } + + if rs.requestPty { + tc := rs.terminalConfig + if tc == nil { + tc = &TerminalConfig{ + Term: "xterm", + Height: 40, + Weight: 80, + } + } + if err := session.RequestPty(tc.Term, tc.Height, tc.Weight, tc.Modes); err != nil { + return err + } + } + + if err := session.Shell(); err != nil { + return err + } + + if err := session.Wait(); err != nil { + return err + } + + return nil +} diff --git a/distribute/wizard.go b/distribute/wizard.go new file mode 100644 index 0000000..74e2b1b --- /dev/null +++ b/distribute/wizard.go @@ -0,0 +1,240 @@ +package distribute + +import ( + "bufio" + "fmt" + "os" + "strings" + "syscall" + + "github.com/Shopify/yaml" + "github.com/fatih/color" + "github.com/j3ssie/osmedeus/libs" + "github.com/j3ssie/osmedeus/provider" + "github.com/j3ssie/osmedeus/utils" + "github.com/olekukonko/tablewriter" + "golang.org/x/term" +) + +func InitCloudSetup(opt libs.Options) { + var supportedProvider = []string{"aws", "digitalocean", "linode"} + fmt.Println("šŸ”® Start cloud setup wizard šŸ”®") + fmt.Println("Currently only these providers are supported: ", color.HiYellowString("%v", supportedProvider)) + + var configProviders provider.ConfigProviders + configProviders.Builder.BuildRepo = opt.Cloud.BuildRepo + if configProviders.Builder.BuildRepo == "" { + configProviders.Builder.BuildRepo = StringPrompt("šŸŒ€ Enter premium install script URL (e.g: https://long-url-here/x/premium.sh)?", "") + } + configProviders.Builder.PublicKey = opt.Cloud.PublicKey + configProviders.Builder.SecretKey = opt.Cloud.SecretKey + + // get the existing one + if opt.Cloud.AddNewProvider { + fmt.Println("šŸŖ„ Add new provider config to the existing config") + providerConfigs, err := provider.ParseProvider(opt.CloudConfigFile) + if err == nil { + configProviders.Clouds = providerConfigs.Clouds + } + } + + for { + configProvider := generateProvider() + configProviders.Clouds = append(configProviders.Clouds, configProvider) + if stop := StringPrompt("šŸŒ€ Do you want to add more provider (y/N)?", "n"); stop != "y" { + break + } + } + + data, err := yaml.Marshal(&configProviders) + if err != nil { + return + } + + //utils.DebugF(string(data)) + if override := StringPrompt("šŸ§™ Do you want to override the old config at "+color.HiGreenString(opt.CloudConfigFile)+" (Y/n)?", "y"); override != "n" { + _, err := utils.WriteToFile(opt.CloudConfigFile, string(data)) + if err != nil { + utils.WarnF("error to write provider config: %v", opt.CloudConfigFile) + } + } + + if isValidate := StringPrompt("ā„ļø Do you want to validate your provider config (Y/n)?", "y"); isValidate == "n" { + return + } + fmt.Printf("šŸ’” You also can manually rebuild the snapshot with the command: %s\n", color.HiCyanString("%s provider build --rebuild", libs.BINARY)) + + var cloudRunners []CloudRunner + for _, configProvider := range configProviders.Clouds { + cloudRunner, err := ValidateProvider(opt, configProvider) + if err != nil { + utils.ErrorF("error validate config: %v -- %v", configProvider.Name, configProvider.RedactedToken) + continue + } + cloudRunners = append(cloudRunners, cloudRunner) + } + + content := [][]string{} + for _, cloudRunner := range cloudRunners { + row := []string{ + cloudRunner.Provider.ProviderName, + cloudRunner.Provider.RedactedToken, + cloudRunner.Provider.SSHKeyID, + cloudRunner.Provider.SnapshotID, + } + content = append(content, row) + } + table := tablewriter.NewWriter(os.Stderr) + table.SetAutoFormatHeaders(false) + table.SetHeader([]string{"Provider", "Token", "SSH Key ID", "Osmedeus Snapshot ID"}) + table.SetBorders(tablewriter.Border{Left: true, Top: false, Right: true, Bottom: false}) + table.SetCenterSeparator("|") + table.AppendBulk(content) // Add Bulk Data + table.Render() + +} + +func generateProvider() provider.ConfigProvider { + configProvider := provider.ConfigProvider{ + Token: "", + Provider: "digitalocean", + DefaultImage: "debian-10-x64", + Size: "s-2vcpu-4gb", + Region: "sfo3", + Username: "root", + Limit: 0, + } + + //# provider: "digitalocean" + //# name: "do-osmp" + //# default_image: "debian-10-x64" + //# size: "s-2vcpu-4gb" + //# region: "sfo3" + + //# provider: "linode" + //# name: "linode-osmp" + //# default_image: "linode/debian10" + //# size: "g6-standard-1" + //# region: "us-east" + + configProvider.Provider = StringPrompt("šŸŒ€ What is your cloud provider?", configProvider.Provider) + + switch configProvider.Provider { + case "do", "digitalocean": + configProvider.Provider = "digitalocean" + fmt.Printf("==> provider selected: %s\n", color.HiBlueString("digitalocean")) + case "ln", "line", "linode": + configProvider = provider.ConfigProvider{ + Token: "", + Provider: "linode", + DefaultImage: "linode/debian10", + Size: "g6-standard-1", + Region: "us-east", + Username: "root", + + Limit: 0, + } + fmt.Printf("==> provider selected: %s\n", color.HiBlueString("linode")) + case "aw", "asw", "aws": + configProvider = provider.ConfigProvider{ + AccessKeyId: "", + SecretKey: "", + Provider: "aws", + DefaultImage: "ami-0ee39036464b9a87e", + Size: "t2.medium", + Region: "ap-southeast-1", + Username: "admin", + Limit: 0, + } + fmt.Printf("==> provider selected: %s\n", color.HiBlueString("aws")) + + default: + configProvider.Provider = "digitalocean" + fmt.Printf("==> provider selected: %s\n", color.HiBlueString("digitalocean")) + } + + if configProvider.Provider == "aws" { + configProvider.AccessKeyId = credentials("AWS AccessKeyId") + fmt.Println() + configProvider.SecretKey = credentials("AWS SecretKey") + fmt.Println() + } else { + configProvider.Token = credentials("API Token") + } + + configProvider.Name = fmt.Sprintf("%s-%s", configProvider.Provider, utils.RandomString(6)) + + if configProvider.Provider == "aws" { + fmt.Printf("šŸ’” Choose your image carefully because it related to region.\n") + fmt.Printf("šŸ’” Refer for this link for more information: " + color.HiCyanString("https://wiki.debian.org/Cloud/AmazonEC2Image/Buster")) + } + configProvider.DefaultImage = StringPrompt("\nšŸŒ€ Choose "+color.HiGreenString("base image")+" for building Osmedeus Image?", configProvider.DefaultImage) + configProvider.Size = StringPrompt("šŸŒ€ Choose "+color.HiGreenString("instance type")+" for running the scan?", configProvider.Size) + configProvider.Region = StringPrompt("šŸŒ€ Choose "+color.HiGreenString("instance region")+" for running the scan?", configProvider.Region) + + return configProvider +} + +func credentials(name string) string { + fmt.Printf("šŸ”‘ Enter your %s? ", color.HiGreenString(name)) + var token string + for { + byteToken, err := term.ReadPassword(int(syscall.Stdin)) + if err == nil && len(byteToken) > 6 { + token = strings.TrimSpace(string(byteToken)) + break + } + utils.WarnF("Looks like your token is invalid. Please try again: %v", token) + } + + redactedToken := token[:5] + "***" + token[len(token)-5:] + fmt.Printf("Your data has been saved: %v", color.HiBlueString(redactedToken)) + return token +} + +// StringPrompt asks for a string value using the label +func StringPrompt(label string, alt string) string { + var s string + r := bufio.NewReader(os.Stdin) + for { + fmt.Fprintf(os.Stderr, fmt.Sprintf("%v (default: %s): ", label, color.HiCyanString(alt))) + s, _ = r.ReadString('\n') + s = strings.TrimSpace(strings.ToLower(s)) + if s == "" { + if alt != "" { + return alt + } + utils.WarnF("Blank input doesn't allow, please specify one") + } + + if s != "" { + break + } + } + return strings.TrimSpace(s) +} + +// ValidateProvider setup new provider +func ValidateProvider(opt libs.Options, providerConfig provider.ConfigProvider) (CloudRunner, error) { + var cloudRunner CloudRunner + cloudRunner.Opt = opt + cloudRunner.Prepare() + + providerCloud, err := provider.InitProviderWithConfig(opt, providerConfig) + if err != nil { + return cloudRunner, err + } + cloudRunner.Provider = providerCloud + + // check if snapshot is okay or not + if !cloudRunner.Provider.SnapshotFound { + utils.InforF("No Snapshot found: %v", cloudRunner.Provider.SnapshotName) + err = cloudRunner.Provider.BuildImage() + if err != nil { + utils.ErrorF("error build snapshot at %v", cloudRunner.Provider.ProviderConfig.BuildFile) + return cloudRunner, err + } + } + + return cloudRunner, nil +} diff --git a/execution/clean.go b/execution/clean.go new file mode 100644 index 0000000..063f10d --- /dev/null +++ b/execution/clean.go @@ -0,0 +1,527 @@ +package execution + +import ( + "bufio" + "fmt" + "net/url" + "os" + "path" + "path/filepath" + "sort" + "strings" + + "github.com/flosch/pongo2/v6" + "github.com/spf13/cast" + + "github.com/thoas/go-funk" + + "github.com/Jeffail/gabs/v2" + "github.com/j3ssie/osmedeus/libs" + "github.com/j3ssie/osmedeus/utils" +) + +// Cleaning the execution directory +func Cleaning(folder string, reports []string) { + utils.DebugF("Cleaning result: %v", folder) + // list all the file + items, err := filepath.Glob(fmt.Sprintf("%v/*", folder)) + if err != nil { + return + } + + for _, item := range items { + item = utils.NormalizePath(item) + utils.DebugF("Check Cleaning: %v", item) + if funk.Contains(reports, item) { + utils.DebugF("Skip cleaning file: %v", item) + continue + } + + fi, err := os.Stat(item) + if err != nil { + continue + } + switch mode := fi.Mode(); { + case mode.IsDir(): + DeleteFolder(item) + continue + case mode.IsRegular(): + DeleteFile(item) + } + } +} + +// CleanGoBuster clean output for gobuster +func CleanGoBuster(src string, output string) { + data := utils.GetFileContent(src) + if data == "" { + return + } + result := strings.Replace(data, "Found: ", "", -1) + utils.WriteToFile(output, result) +} + +// CleanMassdns clean result of massdns to get IP address +func CleanMassdns(filename string, output string) { + file, err := os.Open(utils.NormalizePath(filename)) + if err != nil { + return + } + defer file.Close() + + outputFile, err := os.OpenFile(utils.NormalizePath(output), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + return + } + defer outputFile.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" { + continue + } + if strings.Contains(line, " A ") { + data := strings.Split(line, " A ") + host := strings.Trim(data[0], ".") + ip := strings.Trim(data[1], ".") + outputFile.WriteString(fmt.Sprintf("%v,%v\n", host, ip)) + } else if strings.Contains(line, " CNAME ") { + data := strings.Split(line, " CNAME ") + host := strings.Trim(data[0], ".") + ip := strings.Trim(data[1], ".") + outputFile.WriteString(fmt.Sprintf("%v,%v\n", host, ip)) + } + } +} + +// CleanAmass get IP range and ASN from Amass result +func CleanAmass(filename string, output string) { + content := utils.ReadingLines(filename) + if len(content) <= 0 { + return + } + + var result []string + for _, line := range content { + jsonParsed, err := gabs.ParseJSON([]byte(line)) + if err != nil { + continue + } + asn := jsonParsed.Path("asn").String() + cidr := jsonParsed.Path("cidr").String() + desc := jsonParsed.Path("desc").String() + if cidr != "" && cidr != "null" { + result = append(result, fmt.Sprintf("AS%v,%v,%v", asn, cidr, desc)) + } + } + + if len(result) > 0 { + utils.WriteToFile(output, strings.Join(result, "\n")) + } + +} + +// CleanSWebanalyze get to get formatted report +func CleanSWebanalyze(filename string, output string) { + content := utils.ReadingLines(filename) + if len(content) <= 0 { + return + } + + result := make(map[string][]string) + var finalResult []string + for _, line := range content { + if strings.TrimSpace(line) == "" { + continue + } + if !strings.Contains(line, "tech~") || !strings.Contains(line, "|") { + continue + } + data := strings.Split(line, "|") + domain, _ := utils.GetDomain(strings.TrimLeft(data[1], "~")) + tech := data[2] + if strings.HasSuffix(tech, "/") { + tech = strings.Trim(tech, "/") + } + result[domain] = append(result[domain], tech) + } + + // join summary result + for k, v := range result { + v = funk.UniqString(v) + sort.Strings(v) + techs := strings.Join(v, ",") + data := fmt.Sprintf("domain|%v;;techs|%v", k, strings.Trim(techs, ",")) + finalResult = append(finalResult, data) + } + + if len(result) > 0 { + utils.WriteToFile(output, strings.Join(finalResult, "\n")) + } +} + +// CleanWebanalyze get to get formatted report +func CleanWebanalyze(filename string, output string, sum string) { + content := utils.ReadingLines(filename) + if len(content) <= 0 { + return + } + + result := make(map[string][]string) + var finalResult []string + var techSum []string + for _, line := range content { + jsonParsed, err := gabs.ParseJSON([]byte(line)) + if err != nil { + continue + } + + URL := strings.Trim(jsonParsed.S("hostname").String(), `"`) + u, err := url.Parse(URL) + if err != nil { + continue + } + matches := jsonParsed.S("matches").Children() + for _, child := range matches { + techs := "" + app := strings.Trim(child.S("app_name").String(), `"`) + version := strings.Trim(child.S("version").String(), `"`) + if version != "" { + techs += fmt.Sprintf("%v/%v,", app, version) + } else { + techs += fmt.Sprintf("%v,", app) + } + + // ignore blank techs + if techs == "" || strings.Trim(techs, ",") == "" || strings.Trim(techs, ",") == " " { + continue + } + techSum = append(techSum, strings.Trim(app, ",")) + techs = strings.TrimSpace(strings.Trim(techs, ",")) + result[u.Hostname()] = append(result[u.Hostname()], techs) + } + } + + // join summary result + for k, v := range result { + techs := strings.Join(funk.UniqString(v), ",") + data := fmt.Sprintf("domain|%v;;techs|%v", k, strings.Trim(techs, ",")) + finalResult = append(finalResult, data) + } + + if len(techSum) > 0 { + utils.WriteToFile(sum, strings.Join(funk.UniqString(techSum), "\n")) + } + if len(result) > 0 { + utils.WriteToFile(output, strings.Join(finalResult, "\n")) + } +} + +// CleanArjun clean output of Arjun +func CleanArjun(src string, dest string) { + src = utils.NormalizePath(src) + if !utils.FolderExists(src) { + return + } + + if !strings.HasSuffix(src, "/") { + src += "/" + } + src = path.Join(src, "*") + outputs, err := filepath.Glob(src) + if err != nil { + return + } + + var data []string + for _, output := range outputs { + content := utils.GetFileContent(output) + jsonParsed, err := gabs.ParseJSON([]byte(content)) + if err != nil { + continue + } + prefix := "[GET]" + if strings.HasPrefix(output, "post") { + prefix = "[POST]" + } + + for URL, child := range jsonParsed.ChildrenMap() { + if len(child.Children()) <= 0 { + continue + } + for _, query := range child.Children() { + line := fmt.Sprintf("%v %v?%v=FUZZ", prefix, URL, query.Data()) + data = append(data, line) + } + } + } + + if len(data) > 0 { + utils.WriteToFile(dest, strings.Join(data, "\n")) + } +} + +// CleanJSONDnsx get to get formatted report +func CleanJSONDnsx(filename string, dest string) { + content := utils.ReadingLines(filename) + if len(content) <= 0 { + utils.WarnF("File not found: %s", filename) + return + } + + var results []string + for _, line := range content { + jsonParsed, err := gabs.ParseJSON([]byte(line)) + if err != nil { + continue + } + + domain, ok := jsonParsed.S("host").Data().(string) + if !ok { + continue + } + + a := jsonParsed.S("a") + if a != nil { + for _, record := range a.Children() { + data := fmt.Sprintf("%s A %s", domain, cast.ToString(record.Data())) + results = append(results, data) + } + } + + cname := jsonParsed.S("cname") + if cname != nil { + for _, record := range cname.Children() { + data := fmt.Sprintf("%s CNAME %s", domain, cast.ToString(record.Data())) + results = append(results, data) + } + } + + mx := jsonParsed.S("mx") + if mx != nil { + for _, record := range mx.Children() { + data := fmt.Sprintf("%s MX %s", domain, cast.ToString(record.Data())) + results = append(results, data) + } + } + + ns := jsonParsed.S("ns") + if ns != nil { + for _, record := range ns.Children() { + data := fmt.Sprintf("%s NS %s", domain, cast.ToString(record.Data())) + results = append(results, data) + } + } + + } + + if len(results) > 0 { + utils.WriteToFile(dest, strings.Join(results, "\n")) + } +} + +// CleanRustScan make rustscan data to flat format ip:port +func CleanRustScan(src string, dest string) { + src = utils.NormalizePath(src) + dest = utils.NormalizePath(dest) + content := utils.ReadingLines(src) + if len(content) <= 0 { + utils.WarnF("File not found: %s", src) + return + } + + var results []string + for _, line := range content { + // 103.247.207.76 -> [80,80,443,443] + if !strings.Contains(line, " -> ") { + continue + } + + ip := strings.Split(line, " -> ")[0] + rPorts := strings.Split(line, " -> ")[1] + rPorts = rPorts[1 : len(rPorts)-1] + + if !strings.Contains(rPorts, ",") { + results = append(results, fmt.Sprintf("%s:%s", ip, rPorts)) + } + + ports := strings.Split(rPorts, ",") + ports = funk.UniqString(ports) + for _, port := range ports { + results = append(results, fmt.Sprintf("%s:%s", ip, port)) + } + } + utils.WriteToFile(dest, strings.Join(results, "\n")) +} + +type Vulnerability struct { + SignID string + SignPath string + URL string + Risk string + Confidence string + Request string + ModalID string + + ReportPath string + ReportFile string + + Status string + Length string + Words string + Time string +} + +func GenNucleiReport(opt libs.Options, src string, dest string, templateFile string) { + if templateFile == "" { + templateFile = path.Join(opt.Env.DataFolder, "nuclei-report.html") + } + + if !utils.FileExists(src) { + utils.WarnF("file not found: %v", src) + return + } + content := utils.ReadingLines(src) + var vulns []Vulnerability + + for index, line := range content { + if strings.TrimSpace(line) == "" { + continue + } + jsonParsed, err := gabs.ParseJSON([]byte(line)) + if err != nil { + utils.WarnF("Error parse JSON Data") + continue + } + + modalID, err := utils.GetDomain(cast.ToString(jsonParsed.S("host").Data())) + if err != nil { + continue + } + modalID = fmt.Sprintf("%s-%d", strings.ReplaceAll(modalID, ".", "-"), index) + + vulns = append(vulns, Vulnerability{ + Request: cast.ToString(jsonParsed.S("request").Data()), + URL: cast.ToString(jsonParsed.S("matched-at").Data()), + SignID: cast.ToString(jsonParsed.S("template-id").Data()), + Risk: cast.ToString(jsonParsed.S("info", "severity").Data()), + ModalID: modalID, + Confidence: "Tentative", + }) + } + + utils.DebugF("Reading vuln %v from: %v ", len(vulns), src) + if len(vulns) == 0 { + utils.WarnF("No Vulnerability found %v", src) + return + } + + // read template file + tmpl := utils.GetFileContent(templateFile) + if strings.TrimSpace(tmpl) == "" { + utils.WarnF("empty template data: %v", templateFile) + return + } + + variable := make(map[string]interface{}) + variable["Title"] = "Nuclei Summary Report" + variable["Vulnerabilities"] = vulns + variable["CurrentDay"] = utils.GetCurrentDay() + variable["Version"] = libs.VERSION + variable["Src"] = filepath.Base(src) + + tpl, err := pongo2.FromString(tmpl) + if err != nil { + utils.WarnF("error render data: %v", err) + return + } + out, ok := tpl.Execute(variable) + if ok == nil { + utils.DebugF("Writing Nuclei HTML report to: %v", dest) + utils.WriteToFile(dest, out) + } +} + +// CleanJSONHttpx get to get formatted report +func CleanJSONHttpx(filename string, dest string) { + content := utils.ReadingLines(filename) + if len(content) <= 0 { + utils.WarnF("File not found: %s", filename) + return + } + + var results []string + for _, line := range content { + jsonParsed, err := gabs.ParseJSON([]byte(line)) + if err != nil { + continue + } + + URL := cast.ToString(jsonParsed.S("url").Data()) + //domain, _ := utils.GetDomain(URL) + bodyHash := cast.ToString(jsonParsed.S("body-sha256").Data()) + headerHash := cast.ToString(jsonParsed.S("header-sha256").Data()) + hash := utils.GenHash(fmt.Sprintf("%s-%s", headerHash, bodyHash)) + + title := "No-Title" + rawTitle := jsonParsed.S("title") + if rawTitle != nil { + title = cast.ToString(rawTitle.Data()) + } + + rawTechs := jsonParsed.S("tech") + techs := "No-Tech" + if rawTechs != nil { + techs = strings.Join(cast.ToStringSlice(rawTechs.Data()), ";;") + + } + + data := fmt.Sprintf("%s,%s,%s,%s", URL, hash, title, techs) + results = append(results, data) + + } + + if len(results) > 0 { + utils.WriteToFile(dest, strings.Join(results, "\n")) + } +} + +// CleanFFUFJson get to get formatted report +func CleanFFUFJson(filename string, dest string) { + content := utils.ReadingLines(filename) + if len(content) <= 0 { + utils.WarnF("File not found: %s", filename) + return + } + + var results []string + for _, line := range content { + jsonParsed, err := gabs.ParseJSON([]byte(line)) + if err != nil { + continue + } + + resultsJson := jsonParsed.S("results") + if resultsJson == nil { + continue + } + + for _, item := range resultsJson.Children() { + //.url,.status,.length,.words,.lines,.redirectlocation + endpoint := cast.ToString(item.S("url").Data()) + status := cast.ToString(item.S("status").Data()) + length := cast.ToString(item.S("length").Data()) + words := cast.ToString(item.S("words").Data()) + lines := cast.ToString(item.S("lines").Data()) + redirectLocation := cast.ToString(item.S("redirectlocation").Data()) + + data := fmt.Sprintf("%s,%s,%s,%s,%s,%s", endpoint, status, length, words, lines, redirectLocation) + results = append(results, data) + } + } + + if len(results) > 0 { + utils.WriteToFile(dest, strings.Join(results, "\n")) + } +} diff --git a/execution/clean_test.go b/execution/clean_test.go new file mode 100644 index 0000000..8342b9a --- /dev/null +++ b/execution/clean_test.go @@ -0,0 +1,19 @@ +package execution + +// +//func TestCleanWebanalyze(t *testing.T) { +// var options libs.Options +// options.Debug = true +// CleanWebanalyze("/Users/j3ssie/.osmedeus/workspaces/duckduckgo.com/fingerprint/duckduckgo.com-technology.json", "/Users/j3ssie/.osmedeus/workspaces/duckduckgo.com/fingerprint/duckduckgo.com-technologies.txt") +// +// if !utils.FileExists("/Users/j3ssie/.osmedeus/workspaces/duckduckgo.com/fingerprint/duckduckgo.com-technologies.txt") { +// t.WarnF("Error CleanWebanalyze") +// } +//} + +//func TestCleanSWebanalyze(t *testing.T) { +// CleanSWebanalyze("/tmp/mmm", "/tmp/technologies.txt") +// if !utils.FileExists("/tmp/technologies.txt") { +// t.WarnF("Error CleanSWebanalyze") +// } +//} diff --git a/execution/git.go b/execution/git.go new file mode 100644 index 0000000..03f7206 --- /dev/null +++ b/execution/git.go @@ -0,0 +1,183 @@ +package execution + +import ( + "errors" + "fmt" + "os" + "path" + "strings" + + "github.com/j3ssie/osmedeus/libs" + "github.com/j3ssie/osmedeus/utils" +) + +/* + +Base Git command: +GIT_SSH_COMMAND='ssh -o StrictHostKeyChecking=no -i /path/to/private/key' git xxx + +*/ + +/* +GitDiff('path-to-diff-file', 'path-to-output') +GitDiff('path-to-diff-folder', 'path-to-output') +GitDiff('path-to-diff-file', 'path-to-output', 'number-of-commit-to-diff') +*/ + +// GitDiff run git diff command +func GitDiff(dest string, output string, history string, options libs.Options) { + if options.NoGit || options.Storages["secret_key"] == "" { + return + } + if !utils.FileExists(dest) { + utils.WarnF("File not found: %v", dest) + return + } + utils.DebugF("Git Diff: %v", dest) + diffCmd := fmt.Sprintf("GIT_SSH_COMMAND='ssh -o StrictHostKeyChecking=no -i %v' git diff -U0 HEAD~%v --output=%v %v", options.Storages["secret_key"], history, output, dest) + Execution(diffCmd, options) +} + +// LoopGitDiff like GitDiff but take input as a file +func LoopGitDiff(src string, output string, options libs.Options) { + lines := utils.ReadingFileUnique(src) + if len(lines) == 0 { + return + } + for _, line := range lines { + GitDiff(line, output, "1", options) + } +} + +// DiffCompare run git diff command +func DiffCompare(src string, dest string, output string, options libs.Options) { + if options.NoGit || options.Storages["secret_key"] == "" { + return + } + // if !utils.FileExists(src) || !utils.FileExists(dest) { + // return + // } + diffCmd := fmt.Sprintf("GIT_SSH_COMMAND='ssh -o StrictHostKeyChecking=no -i %v' git diff --no-index --output=%v %v %v", options.Storages["secret_key"], output, src, dest) + Execution(diffCmd, options) +} + +// PullResult pull latest data from result repo +func PullResult(storageFolder string, options libs.Options) { + if options.NoGit || options.Storages["secret_key"] == "" { + return + } + if !utils.FolderExists(storageFolder) { + return + } + utils.DebugF("git pull on: %v", storageFolder) + pullCmd := fmt.Sprintf("GIT_SSH_COMMAND='ssh -o StrictHostKeyChecking=no -i %v' git -C %v pull -f", options.Storages["secret_key"], storageFolder) + Execution(pullCmd, options) +} + +// PushResult push result to git repo +func PushResult(storageFolder string, commitMess string, options libs.Options) { + if options.NoGit || options.Storages["secret_key"] == "" { + return + } + + if !utils.FolderExists(storageFolder) { + return + } + utils.DebugF("git push on: %v", storageFolder) + cmds := []string{ + fmt.Sprintf(`GIT_SSH_COMMAND='ssh -o StrictHostKeyChecking=no -i %v' git -C %v add -A`, options.Storages["secret_key"], storageFolder), + fmt.Sprintf(`GIT_SSH_COMMAND='ssh -o StrictHostKeyChecking=no -i %v' git -C %v commit -m "%v"`, options.Storages["secret_key"], storageFolder, commitMess), + fmt.Sprintf(`GIT_SSH_COMMAND='ssh -o StrictHostKeyChecking=no -i %v' git -C %v push -f`, options.Storages["secret_key"], storageFolder), + fmt.Sprintf(`GIT_SSH_COMMAND='ssh -o StrictHostKeyChecking=no -i %v' git -C %v push -f`, options.Storages["secret_key"], storageFolder), + } + // really run git command + for _, cmd := range cmds { + Execution(cmd, options) + } +} + +// GitClone update latest UI and Plugins from default repo +func GitClone(url string, dest string, forced bool, options libs.Options) { + if options.NoGit || options.Storages["secret_key"] == "" { + utils.WarnF("Storage Disable") + return + } + + if url == "" || dest == "" { + utils.WarnF("Invalid repo or no destination") + return + } + + // check if folder is exist or not + dest = utils.NormalizePath(dest) + if forced { + utils.DebugF("Remove: %v", dest) + os.RemoveAll(dest) + } + + // if folder exist and have .git folder in it, do git pull instead + if utils.FolderExists(dest) { + if utils.FileExists(path.Join(dest, "/.git/HEAD")) { + pullCmd := fmt.Sprintf("GIT_SSH_COMMAND='ssh -o StrictHostKeyChecking=no -i %v' git -C %v pull -f", options.Storages["secret_key"], dest) + Execution(pullCmd, options) + return + } + } + + // cloning new + utils.DebugF("Cloning: %v", url) + cloneCmd := fmt.Sprintf("GIT_SSH_COMMAND='ssh -o StrictHostKeyChecking=no -i %v' git clone --depth=1 %v %v", options.Storages["secret_key"], url, dest) + Execution(cloneCmd, options) +} + +// CloneRepo clone the repo +func CloneRepo(url string, dest string, options libs.Options) error { + if !ValidGitURL(url) { + return errors.New("invalid repo name") + } + if options.NoGit || options.Storages["secret_key"] == "" { + return errors.New("storage Disable") + } + + if url == "" || dest == "" { + return errors.New("storage didn't setup correctly") + } + // check if folder is exist or not + dest = utils.NormalizePath(dest) + utils.DebugF("Cloning: %v", url) + + if utils.FolderExists(dest) { + if utils.FileExists(path.Join(dest, "/.git/HEAD")) { + return nil + } + utils.DebugF("Remove: %v", dest) + os.RemoveAll(dest) + } + + // cloning new + cloneCmd := fmt.Sprintf("GIT_SSH_COMMAND='ssh -o StrictHostKeyChecking=no -i %v' git clone --depth=1 %v %v", options.Storages["secret_key"], url, dest) + Execution(cloneCmd, options) + if utils.FolderExists(dest) { + if utils.FileExists(path.Join(dest, "/.git/HEAD")) { + return nil + } + return nil + } + return errors.New("fail to clone Repo") +} + +// ValidGitURL simple validate git repo +func ValidGitURL(raw string) bool { + if strings.TrimSpace(raw) == "" { + return false + } + if !strings.HasPrefix(raw, "git@") { + return false + } + + if !strings.Contains(raw, "github.com") && !strings.Contains(raw, "gitlab.com") { + return false + } + + return true +} diff --git a/execution/git_test.go b/execution/git_test.go new file mode 100644 index 0000000..6cdbda1 --- /dev/null +++ b/execution/git_test.go @@ -0,0 +1,25 @@ +package execution + +import ( + "fmt" + "testing" + + "github.com/j3ssie/osmedeus/libs" + "github.com/j3ssie/osmedeus/utils" +) + +func TestDiffCompare(t *testing.T) { + var options libs.Options + options.Debug = true + DiffCompare("/Users/j3ssie/.osmedeus/workspaces/duckduckgo.com/subdomain/final-duckduckgo.com.txt", "/Users/j3ssie/.osmedeus/storages/summary/duckduckgo.com/subdomain-duckduckgo.com.txt", "/Users/j3ssie/.osmedeus/workspaces/duckduckgo.com/subdomain/diff-duckduckgo.com-2019-12-28_3:20:9.txt", options) + // DiffCompare(options) + // fmt.Println(result) + + data := utils.GetFileContent("/Users/j3ssie/.osmedeus/workspaces/duckduckgo.com/subdomain/diff-duckduckgo.com-2019-12-28_3:20:9.txt") + fmt.Println(data) + + if !utils.FileExists("/Users/j3ssie/.osmedeus/workspaces/duckduckgo.com/subdomain/diff-duckduckgo.com-2019-12-28_3:20:9.txt") { + t.Logf("Error DiffCompare") + } + +} diff --git a/execution/gitlab.go b/execution/gitlab.go new file mode 100644 index 0000000..00ba0aa --- /dev/null +++ b/execution/gitlab.go @@ -0,0 +1,206 @@ +package execution + +import ( + "errors" + "fmt" + "github.com/j3ssie/osmedeus/libs" + "github.com/j3ssie/osmedeus/utils" + "log" + "strings" + + "github.com/xanzy/go-gitlab" +) + +// GitlabAuth do authentication with gitlab +func GitlabAuth(options libs.Options) (*gitlab.Client, error) { + if options.NoGit { + return nil, nil + } + // prefer using username + if options.Git.Username != "GITLAB_USER" { + utils.DebugF("Do authen with user: %s", options.Git.Username) + git, err := gitlab.NewBasicAuthClient( + options.Git.Username, + options.Git.Password, + gitlab.WithBaseURL(options.Git.BaseURL), + ) + if err != nil { + utils.WarnF("Error authen in Gitlab %v: %v", options.Git.BaseURL, err) + return nil, err + } + return git, nil + } + + if options.Git.Token == "" { + return nil, errors.New("no credentials provided") + } + utils.DebugF("Do authen with token: %s", options.Git.Token) + git, err := gitlab.NewClient( + options.Git.Token, + gitlab.WithBaseURL(options.Git.BaseURL), + ) + if err != nil { + utils.WarnF("Error authen in Gitlab %v: %v", options.Git.BaseURL, err) + return nil, err + } + return git, nil +} + +// CreateGitlabRepo create gitlab repo +func CreateGitlabRepo(repo string, tag string, options libs.Options) string { + if options.NoGit { + return "" + } + git, err := GitlabAuth(options) + if err != nil { + return "" + } + + utils.InforF("Create repo: %v", repo) + tags := []string{options.Git.DefaultTag} + if tag != "" { + if strings.Contains(tag, ",") { + tags = append(tags, strings.Split(tag, ",")...) + } else { + tags = append(tags, tag) + } + } + + opt := gitlab.CreateProjectOptions{ + Name: gitlab.String(repo), + Path: nil, + DefaultBranch: gitlab.String("master"), + Visibility: gitlab.Visibility(gitlab.PrivateVisibility), + TagList: &tags, + } + gRepo, _, err := git.Projects.CreateProject(&opt, nil) + if err != nil { + utils.WarnF("Error to create %v: %v", repo, err) + return "" + } + + optMember := gitlab.AddProjectMemberOptions{ + UserID: gitlab.Int(options.Git.DefaultUID), + AccessLevel: gitlab.AccessLevel(gitlab.MaintainerPermissions), + } + git.ProjectMembers.AddProjectMember(gRepo.ID, &optMember) + utils.InforF("Created repo at: %s with pid: %v", gRepo.SSHURLToRepo, gRepo.ID) + return gRepo.SSHURLToRepo +} + +// DeleteRepo delete repo by id or name +func DeleteRepo(repo string, pid int, options libs.Options) { + if options.NoGit { + return + } + git, err := GitlabAuth(options) + if err != nil { + return + } + // select pid first + if pid != 0 { + utils.InforF("Delete repo with id: %v", pid) + git.Projects.DeleteProject(pid) + return + } + + user, _, err := git.Users.CurrentUser() + projects, _, err := git.Projects.ListUserProjects(user.ID, nil) + for _, project := range projects { + if project.Name == repo { + utils.InforF("Delete repo: %v", repo) + git.Projects.DeleteProject(project.ID) + return + } + } + utils.WarnF("Project not found: %v", repo) +} + +// ListProjects delete repo by id or name +func ListProjects(gitUser int, options libs.Options) { + if options.NoGit { + return + } + git, err := GitlabAuth(options) + if err != nil { + utils.WarnF("Err get do authen user: %v", err) + return + } + uid := gitUser + var username string + if gitUser == 0 { + user, _, err := git.Users.CurrentUser() + if err != nil { + utils.WarnF("Err get current user: %v", err) + return + } + uid = user.ID + username = user.Username + } else { + //user, _, err := git.Users.GetUser(uid, git.Users.) + //if err != nil { + // utils.WarnF("Err get current user: %v", err) + // return + //} + //username = user.Username + } + utils.InforF("Listing projects of uid: %v", uid) + projects, _, err := git.Projects.ListUserProjects(uid, nil) + if err != nil { + utils.WarnF("Error listing projects: %v", err) + return + } + for _, project := range projects { + fmt.Printf("%30s -- %10d\n", fmt.Sprintf("%s/%s", username, project.Name), project.ID) + } +} + +// This example shows how to create a client with username and password. +func GitAuthSample() { + // git, err := gitlab.NewBasicAuthClient( + // "user", + // "password", + // gitlab.WithBaseURL("https://gitlab.com"), + // ) + git, err := gitlab.NewClient( + "token-here-s", + gitlab.WithBaseURL("https://gitlab.com"), + ) + + if err != nil { + log.Fatal(err) + } + + // List all projects + user, _, err := git.Users.CurrentUser() + //spew.Dump(user) + + // Create + projects, _, err := git.Projects.ListUserProjects(user.ID, nil) + ////git.Projects. + //group, _, _ := git.Groups.ListAllGroupMembers(9182310, nil) + //spew.Dump(group) + //tags := []string{"osm", "test"} + ////git.Projects.ListUserProjects() + //opt := gitlab.CreateProjectOptions{ + // Name: gitlab.String("test-project-tag-osm"), + // Path: nil, + // DefaultBranch: gitlab.String("master"), + // //GroupWithProjectTemplatesID: gitlab.Int(9182310), + // Visibility: gitlab.Visibility(gitlab.PrivateVisibility), + // TagList: &tags, + //} + //git.Projects.CreateProject(&opt, nil) + + if err != nil { + log.Fatal(err) + } + + //git.Projects.DeleteProject(pid) + + log.Printf("Found %d projects", len(projects)) + for _, project := range projects { + fmt.Printf("%30s -- %10d\n", fmt.Sprintf("%s/%s", user.Username, project.Name), project.ID) + } + +} diff --git a/execution/gitlab_test.go b/execution/gitlab_test.go new file mode 100644 index 0000000..24050cf --- /dev/null +++ b/execution/gitlab_test.go @@ -0,0 +1,9 @@ +package execution + +import ( + "testing" +) + +func TestGitAuth(t *testing.T) { + GitAuthSample() +} diff --git a/execution/noti.go b/execution/noti.go new file mode 100644 index 0000000..4cf6c03 --- /dev/null +++ b/execution/noti.go @@ -0,0 +1,352 @@ +package execution + +import ( + "encoding/json" + "errors" + "fmt" + "math/rand" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/spf13/cast" + + tgbotapi "github.com/go-telegram-bot-api/telegram-bot-api/v5" + "github.com/j3ssie/osmedeus/libs" + "github.com/j3ssie/osmedeus/utils" + "github.com/robertkrimen/otto" + "github.com/slack-go/slack" +) + +// StatusNoti send to when module is done with report file +func StatusNoti(notiType string, options libs.Options) { + SendAttachment(notiType, "", options) +} + +// ReportNoti send to notification when module is done with report file +func ReportNoti(arguments []otto.Value, options libs.Options) { + if len(arguments) >= 1 { + for _, argument := range arguments { + SendFile(argument.String(), options.Noti.SlackReportChannel, options) + } + return + } + // if doesn't provide report file send all file in noti section + for _, file := range options.Module.Report.Noti { + SendFile(file, options.Noti.SlackReportChannel, options) + } +} + +// DiffNoti send to notification based on diff content +func DiffNoti(arguments []otto.Value, options libs.Options) { + if len(arguments) >= 1 { + for _, argument := range arguments { + filename := argument.String() + if !utils.FileExists(filename) { + continue + } + data := getNewContent(utils.ReadingLines(filename)) + if strings.TrimSpace(data) == "" { + continue + } + // messageContent := fmt.Sprintf("%v \n ```%v```", filename, data) + // SendAttachment("diff", messageContent, options) + SendFile(filename, options.Noti.SlackDiffChannel, options) + } + return + } + // if doesn't provide report file send all file in noti section + for _, filename := range options.Module.Report.Diff { + if !utils.FileExists(filename) { + continue + } + data := getNewContent(utils.ReadingLines(filename)) + if strings.TrimSpace(data) == "" { + continue + } + // messageContent := fmt.Sprintf("%v \n ```%v```", filename, data) + // SendAttachment("diff", messageContent, options) + SendFile(filename, options.Noti.SlackDiffChannel, options) + } +} + +func getNewContent(data []string) string { + var result string + for _, line := range data { + if strings.HasPrefix(line, "+") && !strings.HasPrefix(line, "++") { + result += fmt.Sprintf("%v\n", line) + } + } + return result +} + +// SendAttachment send attach message to specific channel +func SendAttachment(messType string, messContent string, options libs.Options) error { + if options.Noti.SlackToken == "" { + return errors.New("Slack config improperly") + } + + // choose method + var channel, color, mess string + switch messType { + case "start": + channel = options.Noti.SlackStatusChannel + color = "#005b9f" + mess = fmt.Sprintf("%v Start to run *%v* on *%v*", GetEmoji(), options.Module.Name, options.Scan.ROptions["Workspace"]) + break + case "done": + channel = options.Noti.SlackStatusChannel + color = "#32cb00" + mess = fmt.Sprintf("%v Done run *%v* on *%v*", GetEmoji(), options.Module.Name, options.Scan.ROptions["Workspace"]) + break + case "diff": + channel = options.Noti.SlackDiffChannel + color = "#5E35B1" + mess = fmt.Sprintf("%v Diff content on %v: \n %v", GetEmoji(), options.Scan.ROptions["Workspace"], messContent) + break + case "custom": + channel = options.Noti.SlackStatusChannel + color = "#1ABC9C" + mess = messContent + break + } + + if channel == "" || mess == "" { + return errors.New("Slack channel config improperly") + } + utils.DebugF("Sending %v message to %v", messType, channel) + + api := slack.New(options.Noti.SlackToken) + // message config + attachment := slack.Attachment{ + Color: color, + Text: mess, + // sender name + Footer: options.Noti.ClientName, + FooterIcon: GetIcon(), + Ts: json.Number(strconv.FormatInt(time.Now().Unix(), 10)), + } + + _, _, err := api.PostMessage(channel, slack.MsgOptionAttachments(attachment)) + if err != nil { + return err + } + return nil +} + +// SlackWebHook send message with webhook +func SlackWebHook(webhookURL string, content string) error { + content = fmt.Sprintf("```%s```", content) + attachment := slack.Attachment{ + Color: "#1ABC9C", + Text: content, + // sender name + //Footer: options.Noti.ClientName, + FooterIcon: GetIcon(), + Ts: json.Number(strconv.FormatInt(time.Now().Unix(), 10)), + } + + msg := slack.WebhookMessage{ + Attachments: []slack.Attachment{attachment}, + } + err := slack.PostWebhook(webhookURL, &msg) + return err +} + +// WebHookSendAttachment send attach message to specific channel +func WebHookSendAttachment(options libs.Options, messType string, messContent string) error { + if options.NoNoti { + return fmt.Errorf("noti disabled") + } + if options.Noti.SlackWebHook == "" { + return errors.New("slack webhook config improperly") + } + + // choose method + var color, mess string + switch messType { + case "start": + color = "#005b9f" + mess = fmt.Sprintf("%v Start to run *%v* on *%v*", GetEmoji(), options.Module.Name, options.Scan.ROptions["Workspace"]) + break + case "done": + color = "#32cb00" + mess = fmt.Sprintf("%v Done run *%v* on *%v*", GetEmoji(), options.Module.Name, options.Scan.ROptions["Workspace"]) + break + case "diff": + color = "#5E35B1" + mess = fmt.Sprintf("%v Diff content on %v: \n %v", GetEmoji(), options.Scan.ROptions["Workspace"], messContent) + break + case "custom": + color = "#1ABC9C" + mess = messContent + break + default: + color = "#1ABC9C" + mess = messContent + } + + // message config + attachment := slack.Attachment{ + Color: color, + Text: mess, + // sender name + Footer: options.Noti.ClientName, + FooterIcon: GetIcon(), + Ts: json.Number(strconv.FormatInt(time.Now().Unix(), 10)), + } + + msg := slack.WebhookMessage{ + Attachments: []slack.Attachment{attachment}, + } + err := slack.PostWebhook(options.Noti.SlackWebHook, &msg) + if err != nil { + return err + } + return nil +} + +// SendFile send file to specific channel +func SendFile(filename string, channel string, options libs.Options) error { + if options.Noti.SlackToken == "" || options.Noti.SlackReportChannel == "" { + return fmt.Errorf("Slack config improperly") + } + + if !utils.FileExists(filename) { + return fmt.Errorf("report file not found: %v", filename) + } + + baseName := filepath.Base(filename) + mess := fmt.Sprintf("%v - %v - Report file for *%v* on *%v*", GetEmoji(), baseName, options.Module.Name, options.Scan.ROptions["Workspace"]) + utils.DebugF("Sending %v message to %v", filename, channel) + + // sending file + api := slack.New(options.Noti.SlackToken) + params := slack.FileUploadParameters{ + Channels: []string{channel}, + Title: mess, + Filetype: "txt", + File: filename, + } + _, err := api.UploadFile(params) + if err != nil { + return err + } + return nil +} + +// TeleSendMess send message to telegram +func TeleSendMess(options libs.Options, content string, channel string, wrap bool) error { + + if options.NoNoti { + return fmt.Errorf("noti disabled") + } + bot, err := tgbotapi.NewBotAPI(options.Noti.TelegramToken) + content = tgbotapi.EscapeText(tgbotapi.ModeMarkdown, content) + if wrap { + content = fmt.Sprintf("```\n%s\n```", content) + } + if err != nil { + utils.DebugF("error init telegram: %v", err) + return err + } + + if channel == "" || channel == "general" { + channel = options.Noti.TelegramChannel + } + switch channel { + case "#status": + channel = options.Noti.TelegramStatusChannel + case "#r", "#report", "#reports", "#vuln": + channel = options.Noti.TelegramReportChannel + case "#s", "#sensitive", "#sen": + channel = options.Noti.TelegramSensitiveChannel + case "#dirb", "#dirscan": + channel = options.Noti.TelegramDirbChannel + case "#m", "#mics": + channel = options.Noti.TelegramMicsChannel + case "#default", "#general": + channel = options.Noti.TelegramChannel + } + telechannel := cast.ToInt64(channel) + utils.DebugF("send message to channel %v", channel) + msg := tgbotapi.NewMessage(telechannel, content) + msg.ParseMode = "markdown" + _, err = bot.Send(msg) + if err != nil { + utils.DebugF("error sending telegram to %v -- %v", channel, err) + } + return err +} + +// TeleSendFile send message to telegram +func TeleSendFile(options libs.Options, filename string, channel string) error { + if options.NoNoti { + return fmt.Errorf("noti disabled") + } + bot, err := tgbotapi.NewBotAPI(options.Noti.TelegramToken) + if err != nil { + utils.DebugF("error init telegram: %v", err) + return err + } + + if channel == "" || channel == "general" { + channel = options.Noti.TelegramChannel + } + switch channel { + case "#status": + channel = options.Noti.TelegramStatusChannel + case "#r", "#report", "#reports", "#vuln": + channel = options.Noti.TelegramReportChannel + case "#sensitive", "#sen": + channel = options.Noti.TelegramSensitiveChannel + case "#dirb", "#dirscan": + channel = options.Noti.TelegramDirbChannel + case "#m", "#mics": + channel = options.Noti.TelegramMicsChannel + case "#default": + channel = options.Noti.TelegramChannel + } + telechannel := cast.ToInt64(channel) + + filename = utils.NormalizePath(filename) + + msg := tgbotapi.NewDocument(telechannel, tgbotapi.FilePath(filename)) + utils.DebugF("send file %v to channel %v", filename, channel) + _, err = bot.Send(msg) + if err != nil { + utils.DebugF("error sending telegram to %v -- %v", channel, err) + } + return err +} + +/////// utils for slack message + +// GetEmoji get random emoji +func GetEmoji() string { + rand.Seed(time.Now().Unix()) + emojis := []string{ + ":robot_face:", + ":alien:", + ":gift:", + ":gun:", + ":diamond_shape_with_a_dot_inside:", + ":rocket:", + ":bug:", + ":broccoli:", + ":shamrock:", + } + n := rand.Int() % len(emojis) + return emojis[n] +} + +// GetIcon get random emoji +func GetIcon() string { + rand.Seed(time.Now().Unix()) + emojis := []string{ + "https://platform.slack-edge.com/img/default_application_icon.png", + } + n := rand.Int() % len(emojis) + return emojis[n] +} diff --git a/execution/noti_test.go b/execution/noti_test.go new file mode 100644 index 0000000..034a5ca --- /dev/null +++ b/execution/noti_test.go @@ -0,0 +1,44 @@ +package execution + +import ( + "github.com/j3ssie/osmedeus/libs" + "testing" +) + +func TestSlackWebHook(t *testing.T) { + + content := ` +[jira-subversion-xss][Tentative-Medium] - https://142.104.128.133:443/plugins/servlet/svnwebclient/error.jsp?errormessage=%27%22%3E%3Cscript%3Ealert(document.domain)%3C%2Fscript%3E&description=test - out/142.104.128.133/jira-subversion-xss-b38aaad1bc0567262e48e374e902bda60d522f94\n +[jira-subversion-xss][Tentative-Medium] - http://58.49.154.201:8080/plugins/servlet/svnwebclient/error.jsp?errormessage=%27%22%3E%3Cscript%3Ealert(document.domain)%3C%2Fscript%3E&description=test - out/58.49.154.201/jira-subversion-xss-2e18d80689f97485af742d8c0368ffd2cdc78d7a +` + err := SlackWebHook("https://hooks.slack.com/services/T01D9M3RSLA/B01CUUVD98X/xxxxxxxx", content) + if err != nil { + t.Error(err) + } +} + +func TestTeleSendMess(t *testing.T) { + var opt libs.Options + opt.Noti.TelegramChannel = "-1001166523435" + opt.Noti.TelegramToken = "1288534500:xxxxxx" + + content := ` +[jira-subversion-xss][Tentative-Medium] - https://142.104.128.133:443/plugins/servlet/svnwebclient/error.jsp?errormessage=%27%22%3E%3Cscript%3Ealert(document.domain)%3C%2Fscript%3E&description=test - out/142.104.128.133/jira-subversion-xss-b38aaad1bc0567262e48e374e902bda60d522f94\n +[jira-subversion-xss][Tentative-Medium] - http://58.49.154.201:8080/plugins/servlet/svnwebclient/error.jsp?errormessage=%27%22%3E%3Cscript%3Ealert(document.domain)%3C%2Fscript%3E&description=test - out/58.49.154.201/jira-subversion-xss-2e18d80689f97485af742d8c0368ffd2cdc78d7a +` + err := TeleSendMess(opt, content, "general", true) + if err != nil { + t.Error(err) + } +} + +func TestTeleSendFile(t *testing.T) { + var opt libs.Options + opt.Noti.TelegramChannel = "-1001166523435" + opt.Noti.TelegramToken = "1288534500:xxxx" + + err := TeleSendFile(opt, "/tmp/jtt/out/jaeles-summary.txt", "general") + if err != nil { + t.Error(err) + } +} diff --git a/execution/process.go b/execution/process.go new file mode 100644 index 0000000..5d6d879 --- /dev/null +++ b/execution/process.go @@ -0,0 +1,93 @@ +package execution + +import ( + "fmt" + "strings" + "sync" + + "github.com/fatih/color" + "github.com/j3ssie/osmedeus/libs" + gops "github.com/mitchellh/go-ps" + "github.com/panjf2000/ants" + "github.com/shirou/gopsutil/process" + "github.com/spf13/cast" +) + +func ListAllOsmedeusProcess() (pids []int) { + processes, err := gops.Processes() + if err != nil { + return pids + } + var allProcess []OSProcess + + var wg sync.WaitGroup + p, _ := ants.NewPoolWithFunc(20, func(i interface{}) { + defer wg.Done() + + ps := i.(gops.Process) + pid := ps.Pid() + proc, _ := process.NewProcess(cast.ToInt32(pid)) + cmd, _ := proc.Cmdline() + if !strings.Contains(cmd, libs.BINARY) { + return + } + + osProcess := OSProcess{ + PID: pid, + Command: cmd, + } + fmt.Printf("pid:%v %s %v\n", color.HiCyanString("%v", osProcess.PID), color.HiMagentaString("--"), osProcess.Command) + allProcess = append(allProcess, osProcess) + pids = append(pids, osProcess.PID) + }, ants.WithPreAlloc(true)) + defer p.Release() + + for _, ps := range processes { + wg.Add(1) + _ = p.Invoke(ps) + + } + wg.Wait() + + return pids +} + +type OSProcess struct { + PID int `json:"pid"` + Command string `json:"command"` +} + +func GetOsmProcess(processName string) []OSProcess { + if processName == "" { + processName = libs.BINARY + } + var results []OSProcess + processes, err := gops.Processes() + if err != nil { + return results + } + + for _, ps := range processes { + pid := ps.Pid() + binary := ps.Executable() + + if strings.ToLower(binary) != strings.ToLower(processName) { + continue + } + + proc, _ := process.NewProcess(cast.ToInt32(pid)) + cmd, _ := proc.Cmdline() + + if strings.Contains(cmd, fmt.Sprintf("%s utils ps", libs.BINARY)) { + continue + } + + osmProcess := OSProcess{ + PID: pid, + Command: cmd, + } + results = append(results, osmProcess) + } + + return results +} diff --git a/execution/process_test.go b/execution/process_test.go new file mode 100644 index 0000000..4b7b469 --- /dev/null +++ b/execution/process_test.go @@ -0,0 +1,9 @@ +package execution + +import ( + "testing" +) + +func TestListProcess(t *testing.T) { + ListAllOsmedeusProcess() +} diff --git a/execution/remote.go b/execution/remote.go new file mode 100644 index 0000000..d761fbc --- /dev/null +++ b/execution/remote.go @@ -0,0 +1,126 @@ +package execution + +import ( + "fmt" + "net/url" + "path" + "strings" + + "github.com/Jeffail/gabs/v2" + "github.com/j3ssie/osmedeus/libs" + "github.com/j3ssie/osmedeus/utils" + "github.com/parnurzeal/gorequest" +) + +var remoteOptions libs.Options + +// RemoteLogin get to get JWT to run +func RemoteLogin(username string, password string, URL string, options libs.Options) { + u, err := url.Parse(URL) + if err != nil { + return + } + URL = fmt.Sprintf("%v://%v:%v", u.Scheme, u.Hostname(), u.Port()) + + request := gorequest.New() + _, body, _ := request.Post(fmt.Sprintf("%v/auth/login", URL)). + Set("Content-Type", "application/json"). + Send(fmt.Sprintf(`{"username":"%v", "password":"%v"}`, username, password)). + End() + + jsonParsed, _ := gabs.ParseJSON([]byte(body)) + + token := strings.Trim(jsonParsed.S("token").String(), `"`) + if token == "null" { + utils.WarnF("Login fail at %v", URL) + } + options.Client.JWT = fmt.Sprintf("Osmedeus %v", token) + options.Client.URL = URL + utils.DebugF("Got token: %v at %v", options.Client.JWT, options.Client.URL) + remoteOptions = options +} + +// RemoteUpload upload data to remote url +func RemoteUpload(src string, options libs.Options) { + if options.Client.JWT == "" || options.Client.URL == "" { + utils.WarnF("JWT Token was not set") + return + } + // keep \n not escape from JSON body + data := strings.Join(utils.ReadingFileUnique(utils.NormalizePath(src)), "\\n") + filename := path.Base(src) + jsonBody := fmt.Sprintf(`{"data":"%v", "filename":"%v"}`, data, filename) + + request := gorequest.New() + _, body, _ := request.Post(fmt.Sprintf("%v/api/upload/data", options.Client.URL)). + Set("Content-Type", "application/json"). + Set("Authorization", options.Client.JWT). + Send(jsonBody). + End() + // sample --> /tmp/data-osm-sample + jsonParsed, _ := gabs.ParseJSON([]byte(body)) + remoteFile := strings.Trim(jsonParsed.S("content").String(), `"`) + + if remoteFile == "null" { + utils.WarnF("Fail to Upload %v at %v", src, options.Client.URL) + } + utils.DebugF("Sucessfully uploaded: %v", remoteFile) +} + +// RemoteExec run command on a remote client +func RemoteExec(command string, options libs.Options) { + if options.Client.JWT == "" || options.Client.URL == "" { + utils.WarnF("JWT Token was not set") + return + } + + // default master password is blank now + mpassword := "" + request := gorequest.New() + _, body, _ := request.Post(fmt.Sprintf("%v/api/task/new", options.Client.URL)). + Set("Content-Type", "application/json"). + Set("Authorization", options.Client.JWT). + Send(fmt.Sprintf(`{"command": "%v","password": "%v"}`, command, mpassword)). + End() + + jsonParsed, _ := gabs.ParseJSON([]byte(body)) + message := jsonParsed.S("content").String() + if message == "null" { + utils.WarnF("Run Command Fail at %v", options.Client.URL) + } + utils.DebugF("Sucessfully run Command: %v", command) + +} + +// RemoteExecSchedule run command on a remote client with schedule +func RemoteExecSchedule(command string, schedule string, options libs.Options) { + if options.Client.JWT == "" || options.Client.URL == "" { + utils.WarnF("JWT Token was not set") + return + } + + // default master password is blank now + mpassword := "" + request := gorequest.New() + _, body, _ := request.Post(fmt.Sprintf("%v/api/task/new", options.Client.URL)). + Set("Content-Type", "application/json"). + Set("Authorization", options.Client.JWT). + Send(fmt.Sprintf(`{"command":"%v","seconds":%v,"password":"%v"}`, command, schedule, mpassword)). + End() + + jsonParsed, _ := gabs.ParseJSON([]byte(body)) + message := jsonParsed.S("content").String() + if message == "null" { + utils.WarnF("Run Schedule Command Fail at %v", options.Client.URL) + } + utils.DebugF("Sucessfully run Schedule Command: %v", command) + + /* + { + "name": "example", + "command": "id", + "minutes": 1, + "password": "321" + } + */ +} diff --git a/execution/request.go b/execution/request.go new file mode 100644 index 0000000..8d6670b --- /dev/null +++ b/execution/request.go @@ -0,0 +1,227 @@ +package execution + +import ( + "bufio" + "crypto/tls" + "encoding/base64" + "fmt" + "github.com/j3ssie/osmedeus/libs" + "github.com/j3ssie/osmedeus/utils" + "io" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/go-resty/resty/v2" + "github.com/sirupsen/logrus" +) + +func SendGET(token string, url string) (libs.Response, error) { + client := BuildClient(token, 2) + res, err := JustSend(url, client) + return res, err +} + +// BuildClient build base HTTP client +func BuildClient(token string, retry int) *resty.Client { + headers := map[string]string{ + "UserAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36", + "Content-Type": "application/json", + } + if token != "" { + headers["Authorization"] = fmt.Sprintf("Bearer %s", token) + } + + timeout := 15 + + //if len(options.Headers) > 0 { + // for _, head := range options.Headers { + // if strings.Contains(head, ":") { + // data := strings.Split(head, ":") + // if len(data) < 2 { + // continue + // } + // headers[data[0]] = strings.Join(data[1:], "") + // } + // } + //} + + // disable log when retry + logger := logrus.New() + logger.Out = io.Discard + + client := resty.New() + client.SetLogger(logger) + client.SetTransport(&http.Transport{ + MaxIdleConns: 100, + MaxConnsPerHost: 1000, + IdleConnTimeout: time.Duration(timeout) * time.Second, + ExpectContinueTimeout: time.Duration(timeout) * time.Second, + ResponseHeaderTimeout: time.Duration(timeout) * time.Second, + TLSHandshakeTimeout: time.Duration(timeout) * time.Second, + DisableCompression: true, + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }) + + client.SetHeaders(headers) + client.SetCloseConnection(true) + client.SetRetryCount(retry) + client.SetTimeout(time.Duration(timeout) * time.Second) + client.SetRetryWaitTime(time.Duration(timeout/2) * time.Second) + client.SetRetryMaxWaitTime(time.Duration(timeout) * time.Second) + return client +} + +// JustSend just sending request +func JustSend(url string, client *resty.Client) (res libs.Response, err error) { + method := "GET" + // redirect policy + + var resp *resty.Response + // really sending things here + method = strings.ToLower(strings.TrimSpace(method)) + switch method { + case "get": + resp, err = client.R(). + Get(url) + break + case "post": + resp, err = client.R(). + Post(url) + break + } + + // in case we want to get redirect stuff + if res.StatusCode != 0 { + return res, nil + } + + if err != nil || resp == nil { + utils.WarnF("%v %v", url, err) + return libs.Response{}, err + } + + return ParseResponse(*resp), nil +} + +// ParseResponse field to Response +func ParseResponse(resp resty.Response) (res libs.Response) { + // var res libs.Response + resLength := len(string(resp.Body())) + // format the headers + var resHeaders []map[string]string + for k, v := range resp.RawResponse.Header { + if k == "Content-Type" { + res.ContentType = strings.Join(v[:], "") + } + if k == "Location" { + res.Location = strings.Join(v[:], "") + } + element := make(map[string]string) + element[k] = strings.Join(v[:], "") + resLength += len(fmt.Sprintf("%s: %s\n", k, strings.Join(v[:], ""))) + resHeaders = append(resHeaders, element) + } + // response time in second + resTime := float64(resp.Time()) / float64(time.Second) + resHeaders = append(resHeaders, + map[string]string{"Total Length": strconv.Itoa(resLength)}, + map[string]string{"Response Time": fmt.Sprintf("%f", resTime)}, + ) + + // set some variable + res.Headers = resHeaders + res.StatusCode = resp.StatusCode() + res.Status = fmt.Sprintf("%v %v", resp.Status(), resp.RawResponse.Proto) + res.Body = string(resp.Body()) + res.ResponseTime = resTime + res.Length = resLength + // beautify + res.Beautify = BeautifyResponse(res) + res.BeautifyHeader = BeautifyHeaders(res) + return res +} + +// BeautifyRequest beautify request +func BeautifyRequest(req libs.Request) string { + var beautifyReq string + // hardcoded HTTP/1.1 for now + beautifyReq += fmt.Sprintf("%v %v HTTP/1.1\n", req.Method, req.URL) + + for _, header := range req.Headers { + for key, value := range header { + if key != "" && value != "" { + beautifyReq += fmt.Sprintf("%v: %v\n", key, value) + } + } + } + if req.Body != "" { + beautifyReq += fmt.Sprintf("\n%v\n", req.Body) + } + return beautifyReq +} + +// BeautifyHeaders beautify headers +func BeautifyHeaders(res libs.Response) string { + beautifyHeader := fmt.Sprintf("< %v \n", res.Status) + for _, header := range res.Headers { + for key, value := range header { + beautifyHeader += fmt.Sprintf("< %v: %v\n", key, value) + } + } + return beautifyHeader +} + +// BeautifyResponse beautify response +func BeautifyResponse(res libs.Response) string { + var beautifyRes string + beautifyRes += fmt.Sprintf("%v \n", res.Status) + + for _, header := range res.Headers { + for key, value := range header { + beautifyRes += fmt.Sprintf("%v: %v\n", key, value) + } + } + + beautifyRes += fmt.Sprintf("\n%v\n", res.Body) + return beautifyRes +} + +// ParseBurpRequest parse burp style request +func ParseBurpRequest(raw string) string { + rawDecoded, err := base64.StdEncoding.DecodeString(raw) + if err != nil { + return "" + } + + var realReq libs.Request + reader := bufio.NewReader(strings.NewReader(string(rawDecoded))) + parsedReq, err := http.ReadRequest(reader) + if err != nil { + return "" + } + realReq.Method = parsedReq.Method + // URL part + if parsedReq.URL.Host == "" { + realReq.Host = parsedReq.Host + parsedReq.URL.Host = parsedReq.Host + } + if parsedReq.URL.Scheme == "" { + if parsedReq.Referer() == "" { + realReq.Scheme = "https" + parsedReq.URL.Scheme = "https" + } else { + u, err := url.Parse(parsedReq.Referer()) + if err == nil { + realReq.Scheme = u.Scheme + parsedReq.URL.Scheme = u.Scheme + } + } + } + realReq.URL = parsedReq.URL.String() + realReq.Path = parsedReq.RequestURI + + return realReq.URL +} diff --git a/execution/require.go b/execution/require.go new file mode 100644 index 0000000..be9f9a1 --- /dev/null +++ b/execution/require.go @@ -0,0 +1,106 @@ +package execution + +// +//// RunRequire is main function for generator +//func RunRequire(script string, options libs.Options) bool { +// // @NOTE: for some reason < auto translate to < in golang template +// if strings.Contains(script, "<") { +// script = strings.Replace(script, "<", "<", -1) +// } +// utils.DebugF("[Run-Require] %v", script) +// vm := otto.New() +// +// vm.Set("EmptyDir", func(call otto.FunctionCall) otto.Value { +// result, _ := vm.ToValue(utils.EmptyDir(call.Argument(0).String())) +// return result +// }) +// +// vm.Set("NotEmptyDir", func(call otto.FunctionCall) otto.Value { +// result, _ := vm.ToValue(!utils.EmptyDir(call.Argument(0).String())) +// return result +// }) +// +// vm.Set("NotEmptyFile", func(call otto.FunctionCall) otto.Value { +// result, _ := vm.ToValue(!utils.EmptyFile(call.Argument(0).String(), 0)) +// if len(call.ArgumentList) > 1 { +// num, _ := call.Argument(0).ToInteger() +// result, _ = vm.ToValue(!utils.EmptyFile(call.Argument(0).String(), int(num))) +// } +// return result +// }) +// +// vm.Set("EmptyFile", func(call otto.FunctionCall) otto.Value { +// result, _ := vm.ToValue(utils.EmptyFile(call.Argument(0).String(), 0)) +// if len(call.ArgumentList) > 1 { +// num, _ := call.Argument(0).ToInteger() +// result, _ = vm.ToValue(utils.EmptyFile(call.Argument(0).String(), int(num))) +// } +// return result +// }) +// +// vm.Set("ExecContain", func(call otto.FunctionCall) otto.Value { +// var validate bool +// args := call.ArgumentList +// cmd := args[0].String() +// search := args[1].String() +// out, _ := Execution(cmd, options) +// if strings.Contains(out, search) { +// validate = true +// } +// result, _ := vm.ToValue(validate) +// return result +// }) +// +// vm.Set("ExecMatch", func(call otto.FunctionCall) otto.Value { +// var validate bool +// args := call.ArgumentList +// cmd := args[0].String() +// search := args[1].String() +// out, _ := Execution(cmd, options) +// validate = RegexCount(out, search) +// result, _ := vm.ToValue(validate) +// return result +// }) +// +// vm.Set("DirLength", func(call otto.FunctionCall) otto.Value { +// validate := utils.DirLength(call.Argument(0).String()) +// result, err := vm.ToValue(validate) +// if err != nil { +// return otto.FalseValue() +// } +// return result +// }) +// +// vm.Set("FileLength", func(call otto.FunctionCall) otto.Value { +// validate := utils.FileLength(call.Argument(0).String()) +// result, err := vm.ToValue(validate) +// if err != nil { +// return otto.FalseValue() +// } +// return result +// }) +// +// result, serr := vm.Run(script) +// if serr != nil { +// return false +// } +// analyzeResult, err := result.Export() +// if err != nil || analyzeResult == nil { +// return false +// } +// utils.DebugF("Required: %v -- %v", script, result) +// return analyzeResult.(bool) +//} +// +//// RegexCount count regex string in component +//func RegexCount(component string, analyzeString string) bool { +// r, err := regexp.Compile(analyzeString) +// if err != nil { +// return false +// } +// matches := r.FindAllStringIndex(component, -1) +// if len(matches) > 0 { +// return true +// } +// return false +//} diff --git a/execution/s3_cdn.go b/execution/s3_cdn.go new file mode 100644 index 0000000..57d89b5 --- /dev/null +++ b/execution/s3_cdn.go @@ -0,0 +1,125 @@ +package execution + +import ( + "fmt" + "io" + "os" + "path" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/fatih/color" + "github.com/j3ssie/osmedeus/libs" + "github.com/j3ssie/osmedeus/utils" +) + +// UploadToS3 upload local file to s3 bucket +func UploadToS3(options libs.Options, source string, bucket string) { + if options.Cdn.AccessKeyId == "CDN_AWS_ACCESS_KEY" { + return + } + utils.InforF("Uploading %s to %s", color.HiCyanString(source), color.HiCyanString(bucket)) + // Set up a new session and get a reference to the S3 service. + sess, err := session.NewSession(&aws.Config{ + Credentials: credentials.NewStaticCredentials(options.Cdn.AccessKeyId, options.Cdn.SecretKey, ""), + Region: aws.String(options.Cdn.Region)}, + ) + if err != nil { + utils.ErrorF("error creating session: %s", err) + return + } + svc := s3.New(sess) + + if !utils.FileExists(source) { + utils.ErrorF("File not found: %s", source) + return + } + + // Open the file that you want to upload. + file, err := os.Open(source) + if err != nil { + utils.ErrorF("error opening file: %s", err) + return + } + defer file.Close() + + // Set up the parameters for the upload. + params := &s3.PutObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(source), + Body: file, + } + + // Upload the file to S3. + _, err = svc.PutObject(params) + if err != nil { + utils.ErrorF("error uploading file: %s", err) + return + } +} + +// DownloadFromS3 upload local file to s3 bucket +func DownloadFromS3(options libs.Options, source string, dest string, bucket string) { + if options.Cdn.AccessKeyId == "CDN_AWS_ACCESS_KEY" { + return + } + utils.InforF("Downloading %s from %s bucket to %s", color.HiCyanString(source), color.HiCyanString(bucket), color.HiBlueString(dest)) + // Set up a new session and get a reference to the S3 service. + sess, err := session.NewSession(&aws.Config{ + Credentials: credentials.NewStaticCredentials(options.Cdn.AccessKeyId, options.Cdn.SecretKey, ""), + Region: aws.String(options.Cdn.Region)}, + ) + if err != nil { + utils.ErrorF("error creating session: %s", err) + return + } + svc := s3.New(sess) + + // Set up the parameters for the download. + params := &s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(source), + } + + // Download the file from S3. + result, err := svc.GetObject(params) + if err != nil { + utils.ErrorF("error downloading file: %s", err) + return + } + defer result.Body.Close() + + // Create a local file to store the downloaded data. + outFile, err := os.Create(dest) + if err != nil { + utils.ErrorF("error creating local file: %s", err) + return + } + defer outFile.Close() + + // Copy the data from the S3 object to the local file. + _, err = io.Copy(outFile, result.Body) + if err != nil { + utils.ErrorF("error copying file data: %s", err) + return + } +} + +// DownloadFile get file from CDN URL +func DownloadFile(options libs.Options, downloadURL string, dest string) { + utils.DebugF("Downloading: %s", downloadURL) + if !utils.FolderExists(path.Dir(dest)) { + utils.MakeDir(path.Dir(dest)) + } + + cmd := fmt.Sprintf("wget -qO %s %s", dest, downloadURL) + Execution(cmd, options) + + if utils.FileLength(dest) <= 0 { + os.RemoveAll(dest) + return + } + +} diff --git a/execution/scripts.go b/execution/scripts.go new file mode 100644 index 0000000..9b8501c --- /dev/null +++ b/execution/scripts.go @@ -0,0 +1,225 @@ +package execution + +import ( + "archive/tar" + "bufio" + "compress/gzip" + "fmt" + "io" + "os" + "os/exec" + "path" + "regexp" + "sort" + "strings" + "time" + + "github.com/j3ssie/osmedeus/libs" + + "github.com/j3ssie/osmedeus/utils" +) + +// Execution Run a command +func Execution(cmd string, options libs.Options) (string, error) { + command := []string{ + "bash", + "-c", + cmd, + } + //var output string + utils.DebugF("[Exec] %v", command) + realCmd := exec.Command(command[0], command[1:]...) + if options.Quite == true { + realCmd.CombinedOutput() + } else { + // output command output to std too + cmdReader, _ := realCmd.StdoutPipe() + scanner := bufio.NewScanner(cmdReader) + //var out string + go func() { + for scanner.Scan() { + utils.InforF(scanner.Text()) + } + }() + if err := realCmd.Start(); err != nil { + return "", err + } + if err := realCmd.Wait(); err != nil { + return "", err + } + } + return "", nil +} + +// Echo just print testing +func Echo(script string) { + fmt.Println("testing:", script) +} + +// Sleep just print testing +func Sleep(raw string) { + time.Sleep(time.Second * time.Duration(utils.StrToInt(raw))) +} + +// Printf print out some string +func Printf(block string, content string) { + if block != "" { + utils.BlockF(block, content) + } else { + fmt.Println(content) + } +} + +// ErrPrintf print out some string +func ErrPrintf(block string, content string) { + if block != "" { + utils.BadBlockF(block, content) + } else { + fmt.Println(content) + } +} + +// Base just print testing +func Base(raw string) string { + raw = utils.NormalizePath(raw) + return path.Base(raw) +} + +// StripName strip a file name +func StripName(raw string) string { + result := strings.Replace(raw, "/", "_", -1) + result = strings.Replace(result, ":", "_", -1) + return strings.Trim(result, "_") +} + +// DeleteFile delete file +func DeleteFile(filename string) { + utils.DebugF("Delete: %v", filename) + os.Remove(utils.NormalizePath(filename)) +} + +// DeleteFolder delete entire folder +func DeleteFolder(path string) { + utils.DebugF("Delete: %v", path) + os.RemoveAll(utils.NormalizePath(path)) +} + +// Append append content to a file +func Append(dest string, src string) { + if !utils.FileExists(src) || utils.FileLength(src) <= 0 { + utils.DebugF("error to append %v", src) + return + } + data := utils.GetFileContent(src) + utils.AppendToContent(dest, data) +} + +// Copy append content to a file +func Copy(src string, dest string) { + if !utils.FileExists(src) || utils.FileLength(src) <= 0 { + return + } + input, _ := os.ReadFile(src) + os.WriteFile(dest, input, 0644) +} + +// Sort sort content of a file +func Sort(src string) { + data := utils.ReadingFileUnique(src) + if len(data) == 0 { + return + } + sort.Strings(data) + content := strings.Join(data, "\n") + // remove blank line + content = regexp.MustCompile(`[\t\r\n]+`).ReplaceAllString(strings.TrimSpace(content), "\n") + utils.WriteToFile(src, content) +} + +// SortU sort content of a file +func SortU(src string) { + if !utils.FileExists(src) { + utils.DebugF("File not found: %s", src) + } + cmd := fmt.Sprintf("LC_ALL=C sort -u -o %s %s", src, src) + utils.RunCmdWithOutput(cmd) +} + +// Unique unique content of a file and remove blank line +func Unique(filename string) { + data := utils.ReadingFileUnique(filename) + if len(data) > 0 { + content := strings.Join(data, "\n") + // remove blank line + content = regexp.MustCompile(`[\t\r\n]+`).ReplaceAllString(strings.TrimSpace(content), "\n") + utils.WriteToFile(filename, content) + } +} + +// Compress sort content of a file +func Compress(dest string, src string) { + if utils.FolderLength(src) < 1 { + utils.DebugF("Source folder is empty or not found: %s", src) + return + } + cmd := fmt.Sprintf("tar --use-compress-program='gzip -9' -C %s -cf %s .", strings.TrimRight(src, "/"), dest) + utils.RunCmdWithOutput(cmd) +} + +// Decompress sort content of a file +func Decompress(dest string, src string) { + if !utils.FileExists(src) { + utils.DebugF("File not found: %s", src) + return + } + utils.MakeDir(dest) + cmd := fmt.Sprintf("tar -xf %s -C %s", src, dest) + utils.RunCmdWithOutput(cmd) +} + +func ExtractTarGz(filename string) error { + r, err := os.Open(utils.NormalizePath(filename)) + if err != nil { + return err + } + + uncompressedStream, err := gzip.NewReader(r) + if err != nil { + return err + } + + tarReader := tar.NewReader(uncompressedStream) + for true { + header, err := tarReader.Next() + if err == io.EOF { + break + } + + if err != nil { + return err + } + + switch header.Typeflag { + case tar.TypeDir: + if err := os.Mkdir(header.Name, 0755); err != nil { + return err + } + case tar.TypeReg: + outFile, err := os.Create(header.Name) + if err != nil { + return err + } + if _, err := io.Copy(outFile, tarReader); err != nil { + return err + } + outFile.Close() + + default: + return err + + } + + } + + return nil +} diff --git a/execution/scripts_test.go b/execution/scripts_test.go new file mode 100644 index 0000000..8a86937 --- /dev/null +++ b/execution/scripts_test.go @@ -0,0 +1,19 @@ +package execution + +import ( + "testing" +) + +func TestChunkFile(t *testing.T) { + result := ChunkFileByPart("/tmp/oo/seqtest", 3) + t.Log(result) +} + +func TestSort(t *testing.T) { + Sort("/tmp/sam") +} + +func TestIsWildCard(t *testing.T) { + IsWildCard("github.com") + IsWildCard("tesla.com") +} diff --git a/execution/split.go b/execution/split.go new file mode 100644 index 0000000..ee58598 --- /dev/null +++ b/execution/split.go @@ -0,0 +1,126 @@ +package execution + +import ( + "fmt" + "github.com/j3ssie/osmedeus/utils" + "github.com/robertkrimen/otto" + "os" + "path" + "path/filepath" + "strings" +) + +/* + + SplitFile("src", "dest") + SplitFile("src", "base/dest", 500, "another/dest") + next step should be loop in file base/index + +*/ + +// SplitFile split file into multiple file +func SplitFile(kind string, arguments []otto.Value) { + source := arguments[0].String() + dest := arguments[1].String() + // prefix name of the output file + prefix := fmt.Sprintf("%v-chunked", filepath.Base(source)) + destDir := filepath.Dir(utils.NormalizePath(dest)) + if len(arguments) >= 4 { + destDir = arguments[3].String() + } + utils.MakeDir(destDir) + + chunk := 200 + // check if we change the size of it + if len(arguments) >= 3 { + chunkSize, _ := arguments[2].ToInteger() + chunk = int(chunkSize) + } + + chunkParts := chunk + // get number of part + if kind == "size" { + length := utils.FileLength(source) + chunkParts = length / chunk + } + + utils.DebugF("Splitting %v to %v %v", source, chunkParts, kind) + rawChunks, err := utils.SplitLineChunks(source, chunkParts) + if err != nil || len(rawChunks) == 0 { + utils.WarnF("error to split input file: %v", source) + return + } + fp, err := os.Open(source) + if err != nil { + utils.WarnF("error to open input file: %v", source) + return + } + + var sumFile []string + for index, offset := range rawChunks { + targetName := path.Join(destDir, fmt.Sprintf("%v-%v", prefix, index)) + reader := utils.NewRangeReader(fp, offset.Start, offset.Stop) + body := make([]byte, offset.Stop-offset.Start+1) + _, err := reader.Read(body) + if err != nil { + utils.WarnF("error to read chunk file: %s", err) + continue + } + sumFile = append(sumFile, targetName) + utils.DebugF("writing %v part to: %v", index, targetName) + utils.WriteToFile(targetName, string(body)) + } + + // write summary file + indexFile := path.Join(destDir, dest) + _, err = utils.WriteToFile(indexFile, strings.Join(sumFile, "\n")) + if err != nil { + utils.WarnF("Error writing to %v", indexFile) + } +} + +// ChunkFileByPart chunk file to multiple part +func ChunkFileByPart(source string, chunk int) [][]string { + var divided [][]string + data := utils.ReadingLines(source) + if len(data) <= 0 || chunk > len(data) { + if len(data) > 0 { + divided = append(divided, data) + } + return divided + } + + chunkSize := (len(data) + chunk - 1) / chunk + for i := 0; i < len(data); i += chunkSize { + end := i + chunkSize + if end > len(data) { + end = len(data) + } + + divided = append(divided, data[i:end]) + } + return divided +} + +// ChunkFileBySize chunk file to multiple part +func ChunkFileBySize(source string, chunk int) [][]string { + var divided [][]string + data := utils.ReadingLines(source) + if len(data) <= 0 || chunk > len(data) { + if len(data) > 0 { + divided = append(divided, data) + } + return divided + } + + chunkSize := chunk + for i := 0; i < len(data); i += chunkSize { + end := i + chunkSize + if end > len(data) { + end = len(data) + } + + divided = append(divided, data[i:end]) + } + return divided +} diff --git a/execution/wildcard.go b/execution/wildcard.go new file mode 100644 index 0000000..9573c3c --- /dev/null +++ b/execution/wildcard.go @@ -0,0 +1,61 @@ +package execution + +import ( + "fmt" + + //"github.com/OWASP/Amass/v3/requests" + //amassresolvers "github.com/OWASP/Amass/v3/resolvers" + "github.com/j3ssie/osmedeus/utils" +) + +var defaultResolvers = []string{ + "1.1.1.1:53", // Cloudflare + "8.8.8.8:53", // Google + "64.6.64.6:53", // Verisign + "8.8.4.4:53", // Google Secondary +} + +// IsWildCard check if target is wildcard or not +func IsWildCard(domain string) bool { + //var resolvers []string + //resolvers = defaultResolvers + //resolverPool := amassresolvers.SetupResolverPool(resolvers, 1000, false, nil) + //if resolverPool == nil { + // utils.WarnF("Failed to init DNS pool") + // return false + //} + // + //ctx := context.Background() + //defer ctx.Done() + //subdomains := genSubs(domain, 5) + //var totalWildCard int + //for _, subdomain := range subdomains { + // req := &requests.DNSRequest{ + // Name: subdomain, + // Domain: domain, + // } + // if !resolverPool.MatchesWildcard(ctx, req) { + // utils.DebugF("[wild] %s\n", req.Name) + // } else { + // totalWildCard += 1 + // utils.DebugF("[wild] %s\n", req.Name) + // } + //} + //utils.DebugF("Total number of wildcard: %v/%v\n", totalWildCard, len(subdomains)) + //if totalWildCard == len(subdomains) { + // utils.DebugF("Target %v is wildcard\n", domain) + // return true + //} + return false +} + +func genSubs(domain string, size int) []string { + var subdomains []string + subdomains = append(subdomains, fmt.Sprintf("notj3ssei.%s", domain)) + subdomains = append(subdomains, fmt.Sprintf("verylong.%s", domain)) + + for i := 0; i < (size - 2); i++ { + subdomains = append(subdomains, fmt.Sprintf("%s.%s", utils.RandomString(5), domain)) + } + return subdomains +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..fb6c04f --- /dev/null +++ b/go.mod @@ -0,0 +1,113 @@ +module github.com/j3ssie/osmedeus + +go 1.20 + +require ( + github.com/Jeffail/gabs/v2 v2.7.0 + github.com/Shopify/yaml v2.1.0+incompatible + github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 + github.com/aws/aws-sdk-go v1.44.327 + github.com/cenkalti/backoff/v4 v4.2.1 + github.com/davecgh/go-spew v1.1.1 + github.com/dgrijalva/jwt-go v3.2.0+incompatible + github.com/digitalocean/godo v1.102.1 + github.com/fatih/color v1.15.0 + github.com/flosch/pongo2/v6 v6.0.0 + github.com/fsnotify/fsnotify v1.6.0 + github.com/go-playground/validator/v10 v10.15.1 + github.com/go-resty/resty/v2 v2.7.0 + github.com/go-telegram-bot-api/telegram-bot-api/v5 v5.5.1 + github.com/gofiber/fiber/v2 v2.48.0 + github.com/gofiber/jwt/v2 v2.2.7 + github.com/gomarkdown/markdown v0.0.0-20230716120725-531d2d74bc12 + github.com/hashicorp/go-version v1.6.0 + github.com/jasonlvhit/gocron v0.0.1 + github.com/jinzhu/copier v0.4.0 + github.com/json-iterator/go v1.1.12 + github.com/kyokomi/emoji v2.2.4+incompatible + github.com/linode/linodego v1.20.1 + github.com/mackerelio/go-osstat v0.2.4 + github.com/mitchellh/go-homedir v1.1.0 + github.com/mitchellh/go-ps v1.0.0 + github.com/olekukonko/tablewriter v0.0.5 + github.com/panjf2000/ants v1.3.0 + github.com/parnurzeal/gorequest v0.2.16 + github.com/robertkrimen/otto v0.2.1 + github.com/shirou/gopsutil v3.21.11+incompatible + github.com/sirupsen/logrus v1.9.3 + github.com/slack-go/slack v0.12.2 + github.com/spf13/cast v1.5.1 + github.com/spf13/cobra v1.7.0 + github.com/spf13/viper v1.16.0 + github.com/swaggo/swag v1.16.1 + github.com/thoas/go-funk v0.9.3 + github.com/x-cray/logrus-prefixed-formatter v0.5.2 + github.com/xanzy/go-gitlab v0.90.0 + golang.org/x/crypto v0.12.0 + golang.org/x/net v0.14.0 + golang.org/x/oauth2 v0.11.0 + golang.org/x/term v0.11.0 + golang.org/x/text v0.12.0 +) + +require ( + github.com/KyleBanks/depth v1.2.1 // indirect + github.com/PuerkitoBio/purell v1.1.1 // indirect + github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect + github.com/andybalholm/brotli v1.0.5 // indirect + github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a // indirect + github.com/gabriel-vasile/mimetype v1.4.2 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect + github.com/go-openapi/jsonpointer v0.19.5 // indirect + github.com/go-openapi/jsonreference v0.19.6 // indirect + github.com/go-openapi/spec v0.20.4 // indirect + github.com/go-openapi/swag v0.19.15 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/golang-jwt/jwt/v4 v4.0.0 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/go-querystring v1.1.0 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/gorilla/websocket v1.4.2 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-retryablehttp v0.7.4 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/klauspost/compress v1.16.3 // indirect + github.com/leodido/go-urn v1.2.4 // indirect + github.com/magiconair/properties v1.8.7 // indirect + github.com/mailru/easyjson v0.7.6 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect + github.com/mattn/go-runewidth v0.0.14 // indirect + github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pelletier/go-toml/v2 v2.0.8 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/rivo/uniseg v0.2.0 // indirect + github.com/smartystreets/goconvey v1.8.1 // indirect + github.com/spf13/afero v1.9.5 // indirect + github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/subosito/gotenv v1.4.2 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/valyala/fasthttp v1.48.0 // indirect + github.com/valyala/tcplisten v1.0.0 // indirect + github.com/yusufpapurcu/wmi v1.2.3 // indirect + golang.org/x/sys v0.11.0 // indirect + golang.org/x/time v0.3.0 // indirect + golang.org/x/tools v0.7.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.31.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/sourcemap.v1 v1.0.5 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + moul.io/http2curl v1.0.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..cd51a50 --- /dev/null +++ b/go.sum @@ -0,0 +1,737 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Jeffail/gabs/v2 v2.7.0 h1:Y2edYaTcE8ZpRsR2AtmPu5xQdFDIthFG0jYhu5PY8kg= +github.com/Jeffail/gabs/v2 v2.7.0/go.mod h1:dp5ocw1FvBBQYssgHsG7I1WYsiLRtkUaB1FEtSwvNUw= +github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= +github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= +github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/Shopify/yaml v2.1.0+incompatible h1:Y7Wj6eQo5Byeaxb2M9FiPdKBQ3ooLsRIEmUZMMxCI8g= +github.com/Shopify/yaml v2.1.0+incompatible/go.mod h1:Mrsv1G5Osez+VdHYcSI2zfaUzPu4lCdO9cW8R2lHPEc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/andybalholm/brotli v1.0.2/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= +github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= +github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/aws/aws-sdk-go v1.44.327 h1:ZS8oO4+7MOBLhkdwIhgtVeDzCeWOlTfKJS7EgggbIEY= +github.com/aws/aws-sdk-go v1.44.327/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/digitalocean/godo v1.102.1 h1:BrNePwIXjQWjOJXVTBqkURMjm70BRR0qXbRKfHNBF24= +github.com/digitalocean/godo v1.102.1/go.mod h1:SaUYccN7r+CO1QtsbXGypAsgobDrmSfVMJESEfXgoEg= +github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= +github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= +github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= +github.com/flosch/pongo2/v6 v6.0.0 h1:lsGru8IAzHgIAw6H2m4PCyleO58I40ow6apih0WprMU= +github.com/flosch/pongo2/v6 v6.0.0/go.mod h1:CuDpFm47R0uGGE7z13/tTlt1Y6zdxvr2RLT5LJhsHEU= +github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= +github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.19.6 h1:UBIxjkht+AWIgYzCDSv2GN+E/togfwXUJFRTWhl2Jjs= +github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= +github.com/go-openapi/spec v0.20.4 h1:O8hJrt0UMnhHcluhIdUgCLRWyM2x7QkBXRvOs7m+O1M= +github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM= +github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.15.1 h1:BSe8uhN+xQ4r5guV/ywQI4gO59C2raYcGffYWZEjZzM= +github.com/go-playground/validator/v10 v10.15.1/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= +github.com/go-redis/redis v6.15.5+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= +github.com/go-resty/resty/v2 v2.7.0 h1:me+K9p3uhSmXtrBZ4k9jcEAfJmuC8IivWHwaLZwPrFY= +github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I= +github.com/go-telegram-bot-api/telegram-bot-api/v5 v5.5.1 h1:wG8n/XJQ07TmjbITcGiUaOtXxdrINDz1b0J1w0SzqDc= +github.com/go-telegram-bot-api/telegram-bot-api/v5 v5.5.1/go.mod h1:A2S0CWkNylc2phvKXWBBdD3K0iGnDBGbzRpISP2zBl8= +github.com/go-test/deep v1.0.4 h1:u2CU3YKy9I2pmu9pX0eq50wCgjfGIt539SqR7FbHiho= +github.com/go-test/deep v1.0.4/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= +github.com/gofiber/fiber/v2 v2.17.0/go.mod h1:iftruuHGkRYGEXVISmdD7HTYWyfS2Bh+Dkfq4n/1Owg= +github.com/gofiber/fiber/v2 v2.48.0 h1:cRVMCb9aUJDsyHxGFLwz/sGzDggdailZZyptU9F9cU0= +github.com/gofiber/fiber/v2 v2.48.0/go.mod h1:xqJgfqrc23FJuqGOW6DVgi3HyZEm2Mn9pRqUb2kHSX8= +github.com/gofiber/jwt/v2 v2.2.7 h1:MgXZV+ak+FiRVepD3btHBxWcyxlFzTDGXJv78dU1sIE= +github.com/gofiber/jwt/v2 v2.2.7/go.mod h1:yaOHLccYXJidk1HX/EiIdIL+Z1xmY2wnIv6hgViw384= +github.com/golang-jwt/jwt/v4 v4.0.0 h1:RAqyYixv1p7uEnocuy8P1nru5wprCh/MH2BIlW5z5/o= +github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gomarkdown/markdown v0.0.0-20230716120725-531d2d74bc12 h1:uK3X/2mt4tbSGoHvbLBHUny7CKiuwUip3MArtukol4E= +github.com/gomarkdown/markdown v0.0.0-20230716120725-531d2d74bc12/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= +github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-hclog v1.2.0 h1:La19f8d7WIlm4ogzNHB0JGqs5AUDAZ2UfCY4sJXcJdM= +github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA= +github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= +github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= +github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jasonlvhit/gocron v0.0.1 h1:qTt5qF3b3srDjeOIR4Le1LfeyvoYzJlYpqvG7tJX5YU= +github.com/jasonlvhit/gocron v0.0.1/go.mod h1:k9a3TV8VcU73XZxfVHCHWMWF9SOqgoku0/QlY2yvlA4= +github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8= +github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/compress v1.16.3 h1:XuJt9zzcnaz6a16/OU53ZjWp/v7/42WcR5t2a0PcNQY= +github.com/klauspost/compress v1.16.3/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kyokomi/emoji v2.2.4+incompatible h1:np0woGKwx9LiHAQmwZx79Oc0rHpNw3o+3evou4BEPv4= +github.com/kyokomi/emoji v2.2.4+incompatible/go.mod h1:mZ6aGCD7yk8j6QY6KICwnZ2pxoszVseX1DNoGtU2tBA= +github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= +github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= +github.com/linode/linodego v1.20.1 h1:IW3SrZjRzrclYZnzFd80f8lSkTYAM7gVTJW0t7HnFKQ= +github.com/linode/linodego v1.20.1/go.mod h1:ggoWnJXssx9wPWNnR3x7WaOpOBOEhsPB/HO7iflF5qY= +github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= +github.com/mackerelio/go-osstat v0.2.4/go.mod h1:Zy+qzGdZs3A9cuIqmgbJvwbmLQH9dJvtio5ZjJTbdlQ= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= +github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= +github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI= +github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc= +github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= +github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/panjf2000/ants v1.3.0 h1:8pQ+8leaLc9lys2viEEr8md0U4RN6uOSUCE9bOYjQ9M= +github.com/panjf2000/ants v1.3.0/go.mod h1:AaACblRPzq35m1g3enqYcxspbbiOJJYaxU2wMpm1cXY= +github.com/parnurzeal/gorequest v0.2.16 h1:T/5x+/4BT+nj+3eSknXmCTnEVGSzFzPGdpqmUVVZXHQ= +github.com/parnurzeal/gorequest v0.2.16/go.mod h1:3Kh2QUMJoqw3icWAecsyzkpY7UzRfDhbRdTjtNwNiUE= +github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= +github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/robertkrimen/otto v0.2.1 h1:FVP0PJ0AHIjC+N4pKCG9yCDz6LHNPCwi/GKID5pGGF0= +github.com/robertkrimen/otto v0.2.1/go.mod h1:UPwtJ1Xu7JrLcZjNWN8orJaM5n5YEtqL//farB5FlRY= +github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= +github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/slack-go/slack v0.12.2 h1:x3OppyMyGIbbiyFhsBmpf9pwkUzMhthJMRNmNlA4LaQ= +github.com/slack-go/slack v0.12.2/go.mod h1:hlGi5oXA+Gt+yWTPP0plCdRKmjsDxecdHxYQdlMQKOw= +github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGByCY= +github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY= +github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60= +github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM= +github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= +github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= +github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.16.0 h1:rGGH0XDZhdUOryiDWjmIvUSWpbNqisK8Wk0Vyefw8hc= +github.com/spf13/viper v1.16.0/go.mod h1:yg78JgCJcbrQOvV9YLXgkLaZqUidkY9K+Dd1FofRzQg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= +github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= +github.com/swaggo/swag v1.16.1 h1:fTNRhKstPKxcnoKsytm4sahr8FaYzUcT7i1/3nd/fBg= +github.com/swaggo/swag v1.16.1/go.mod h1:9/LMvHycG3NFHfR6LwvikHv5iFvmPADQ359cKikGxto= +github.com/thoas/go-funk v0.9.3 h1:7+nAEx3kn5ZJcnDm2Bh23N2yOtweO14bi//dvRtgLpw= +github.com/thoas/go-funk v0.9.3/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.26.0/go.mod h1:cmWIqlu99AO/RKcp1HWaViTqc57FswJOfYYdPJBl8BA= +github.com/valyala/fasthttp v1.48.0 h1:oJWvHb9BIZToTQS3MuQ2R3bJZiNSa2KiNdeI8A+79Tc= +github.com/valyala/fasthttp v1.48.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA= +github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= +github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= +github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJzfthRT6usrui8uGmg= +github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= +github.com/xanzy/go-gitlab v0.90.0 h1:j8ZUHfLfXdnC+B8njeNaW/kM44c1zw8fiuNj7D+qQN8= +github.com/xanzy/go-gitlab v0.90.0/go.mod h1:5ryv+MnpZStBH8I/77HuQBsMbBGANtVpLWC15qOjWAw= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= +github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= +golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= +golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= +golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= +golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/sourcemap.v1 v1.0.5 h1:inv58fC9f9J3TK2Y2R1NPntXEn3/wjWHkonhIUODNTI= +gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +moul.io/http2curl v1.0.0 h1:6XwpyZOYsgZJrU8exnG87ncVkU1FVCcTRpwzOkTDUi8= +moul.io/http2curl v1.0.0/go.mod h1:f6cULg+e4Md/oW1cYmwW4IWQOVl2lGbmCNGOHvzX2kE= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/libs/cloud.go b/libs/cloud.go new file mode 100644 index 0000000..ce9ec54 --- /dev/null +++ b/libs/cloud.go @@ -0,0 +1,115 @@ +package libs + +// Cloud struct define folder to push data +type Cloud struct { + CheckingLimit bool + ReBuildBaseImage bool + IgnoreConfigFile bool + BackgroundRun bool + OnlyCreateDroplet bool + OnlyCreateInstance bool + EnablePrivateIP bool + + EnableSyncWorkflow bool + AddNewProvider bool + RemoteWorkflowFolder string + TokensFile string + + CopyWorkspaceToGit bool + ClearTime string + InstanceName string + TempTarget string + // content of secret key to avoid reading it too much + SecretKeyContent string + PublicKeyContent string + + // enable terraform + EnableTerraform bool + + // chunk options + ChunkInputs string + BaseWorkspace string + LocalSyncFolder string + DisableLocalSync bool + RemoteRunList bool + TargetAsFile bool + EnableChunk bool + IgnoreProcess bool + NumberOfParts int + Threads int + + // specific cloud instance resources + Size string + Region string + Token string + Provider string + IgnoreSetup bool + + // for pre-commands and post-commands + RemotePreRun []string + // run script on local machine after scan done + LocalSteps []Step `yaml:"local_steps"` + LocalPreRun []string + LocalPostRun []string + + // use to clone build-osm repo + SecretKey string + PublicKey string + BuildRepo string + Binary string + Retry int + UnzipResult bool + + // for health check + ForEverHealthCheck bool + NoDelete bool + + // raw command here + Extra string + Flow string + Module string + Workspace string + RawCommand string + Params []string + + WsSource string + WsDest string + + Input string + Inputs []string + InputsFile string + + Target map[string]string +} + +// Request all information about request +type Request struct { + Timeout int + Repeat int + Scheme string + Host string + Port string + Path string + URL string + Proxy string + Method string + Redirect bool + Headers []map[string]string + Body string + Beautify string +} + +// Response all information about response +type Response struct { + HasPopUp bool + StatusCode int + Status string + ContentType string + Headers []map[string]string + Body string + ResponseTime float64 + Length int + Beautify string + Location string + BeautifyHeader string +} diff --git a/libs/flow.go b/libs/flow.go new file mode 100644 index 0000000..762fc16 --- /dev/null +++ b/libs/flow.go @@ -0,0 +1,69 @@ +package libs + +// Routine for each scan +type Routine struct { + RoutineName string + FlowFolder string `yaml:"flow"` + Timeout string `yaml:"timeout"` + ParsedModules []Module + Modules []string +} + +// Flow struct to define specific field for a mode +type Flow struct { + NoDB bool `yaml:"nodb"` + SkipIndexed bool `yaml:"skip-indexed"` + ForceParams bool `yaml:"force-params"` + Input string + Validator string // domain, cidr, ip or domain-file, cidr-file and so on + + Name string + Type string + DefaultType string + Desc string + Usage string + + Params []map[string]string + Routines []Routine + + RemotePreRun []string `yaml:"remote_pre_run"` + // run script on local machine after scan done + LocalPreRun []string `yaml:"local_pre_run"` + LocalPostRun []string `yaml:"local_post_run"` +} + +// Module struct to define specific field for a module +type Module struct { + NoDB bool `yaml:"nodb"` + Validator string // domain, cidr, ip + ForceParams bool `yaml:"force-params"` + + // just for print some info + Name string + Desc string + Usage string + + // enable resume, if all reports file exist then skip the module + Resume bool + // run module despite resume enable + Forced bool + + MTimeout string `yaml:"mtimeout"` + Params []map[string]string + ModulePath string + + PreRun []string `yaml:"pre_run"` + Report struct { + Final []string + Noti []string + Diff []string + } + Steps []Step + PostRun []string `yaml:"post_run"` + + RemotePreRun []string `yaml:"remote_pre_run"` + // run script on local machine after scan done + LocalSteps []Step `yaml:"local_steps"` + LocalPreRun []string `yaml:"local_pre_run"` + LocalPostRun []string `yaml:"local_post_run"` +} diff --git a/libs/mics.go b/libs/mics.go new file mode 100644 index 0000000..af3fc70 --- /dev/null +++ b/libs/mics.go @@ -0,0 +1,61 @@ +package libs + +// Cdn credentials for other client +type Cdn struct { + Bucket string + Region string + SecretKey string + AccessKeyId string +} + +// Git credentials for other client +type Git struct { + BaseURL string + Username string + Password string + Token string + Group string + DefaultPrefix string + DefaultTag string + DefaultUser string + DefaultUID int + DeStorage string +} + +// TmuxOpt credentials for other client +type TmuxOpt struct { + ApplyAll bool + SelectedWindow string + Exclude string + Limit int +} + +// Cron credentials for other client +type Cron struct { + Command string + Schedule int + Forever bool +} + +// Remote credentials for other client +type Remote struct { + MasterHost string + MasterCred string + PoolHost string + PoolCred string +} + +// Sync credentials for other client +type Sync struct { + BaseURL string + Prefix string + Pool string +} + +// Client credentials for other client +type Client struct { + Username string + Password string + JWT string + URL string +} diff --git a/libs/noti.go b/libs/noti.go new file mode 100644 index 0000000..4ebba47 --- /dev/null +++ b/libs/noti.go @@ -0,0 +1,26 @@ +package libs + +// Notification struct define notification method +type Notification struct { + ClientName string + // SlacksWebHooks list + SlacksWebHooks map[string]string + // TelegramWebHooks list + TelegramWebHooks map[string]string + // Telegram part + TelegramToken string + TelegramChannel string + TelegramStatusChannel string + TelegramReportChannel string + TelegramDirbChannel string + TelegramSensitiveChannel string + TelegramMicsChannel string + // use this when we want to send a file to channel + SlackWebHook string + SlackToken string + SlackReportChannel string + SlackStatusChannel string + SlackDiffChannel string + // later then + DiscordToken string +} diff --git a/libs/options.go b/libs/options.go new file mode 100644 index 0000000..d5c082a --- /dev/null +++ b/libs/options.go @@ -0,0 +1,172 @@ +package libs + +// Options global options +type Options struct { + ConfigFile string + LogFile string + Concurrency int + + // default threads hold for running module + Tactics string + Threads int + + Timeout string + EnableFormatInput bool + Verbose bool + FullHelp bool + + // some disable options + NoPostRun bool + NoPreRun bool + NoNoti bool + NoBanner bool + NoGit bool + NoClean bool + NoDB bool + NoCdn bool + DisableValidateInput bool + + PremiumPackage bool + Resume bool + Quite bool + Force bool + WildCardCheck bool + Debug bool + EnableDeStorage bool + PID int + SyncTimes int + PollingTime int + MDCodeBlockLimit int + Exclude []string + Params []string + CustomGit bool + EnableBackup bool + JsonOutput bool + + Client Client + Queue Queue + Git Git + Sync Sync + Scan Scan + Server Server + Env Environment + Noti Notification + Flow Flow + Module Module + Tmux TmuxOpt + Cron Cron + Remote Remote + Cdn Cdn + Update Update + + ThreadsHold ThreadsHold + Cloud Cloud + Report Report + CloudConfigFile string + TokenConfigFile string + GitSync bool + + ScanID string + Storages map[string]string +} + +// Scan sub options for scan +type Scan struct { + ROptions map[string]string + Params []string + Input string + InputType string // domain, url, ip, cidr or domainList, urlList, ipList, cidrList + ParamsFile string + Inputs []string + InputList string + Modules []string + Flow string + + BaseWorkspace string + CustomWorkspace string + SuffixName string + Force bool + // this is true when calling from cloud scan + RemoteCall bool +} + +type ThreadsHold struct { + Default int + Aggressive int + Gently int +} + +// Report sub options for report +type Report struct { + CustomPreFix string + PublicIP string + ExtractFolder string + Static bool + Raw bool +} + +// Server sub options for api server +type Server struct { + DisableWorkspaceListing bool + DisableSSL bool + PreFork bool + NoAuthen bool + + PollingTime int + Bind string + Port string + StaticPrefix string + JWTSecret string + Cors string + UIPath string + MasterPassword string + + // database + DBPath string + DBType string + DBConnection string + DBName string + DBUser string + DBPass string + DBHost string + DBPort string + + // for SSL + CertFile string + KeyFile string +} + +// Storage struct define folder to push data +type Storage struct { + SecretKey string + SummaryStorage string + SummaryRepo string + HTTPStorage string + HTTPRepo string + AssetsStorage string + AssetsRepo string +} + +// Environment some config path +type Environment struct { + RootFolder string // ~/.osmedeus + StoragesFolder string // ~/.osmedeus/storages/ + WorkspacesFolder string // ~/workspaces-osmedeus/ + + // Base one + BaseFolder string // ~/osmedeus-base + BinariesFolder string // ~/osmedeus-base/binaries + DataFolder string // ~/osmedeus-base/data/ + OseFolder string // ~/osmedeus-base/ose/ + WorkFlowsFolder string // ~/osmedeus-base/workflow/ + + // cloud stuff + CloudConfigFolder string // ~/osmedeus-base/clouds/ + ProviderFolder string // ~/.osmedeus/providers/ + InstancesFolder string // ~/.osmedeus/instances/ + BackupFolder string + + // Mics + ScriptsFolder string + UIFolder string +} diff --git a/libs/queue.go b/libs/queue.go new file mode 100644 index 0000000..62065ac --- /dev/null +++ b/libs/queue.go @@ -0,0 +1,22 @@ +package libs + +// Queue sub options for quque +type Queue struct { + QueueFolder string + QueueFile string + RawCommand string + + InputAsFile bool + Add bool +} + +type InputFormat struct { + Input string `json:"input"` + Flow string `json:"flow"` + Modules []string `json:"module"` + Params []string `json:"params"` + Workspaces string `json:"workspace"` + Extra string `json:"extra"` + Command string `json:"command"` + InputAsFile bool `json:"input-as-file"` +} diff --git a/libs/step.go b/libs/step.go new file mode 100644 index 0000000..8ccd353 --- /dev/null +++ b/libs/step.go @@ -0,0 +1,30 @@ +package libs + +// Step struct to define component about a command +type Step struct { + // timeout for commands and script + Timeout string + // use for run loop command + Parallel int + Threads string + Source string + + Label string + + Conditions []string + Required []string + + Commands []string + Ose []string `yaml:"ose"` + Scripts []string + + // run when conditions are false + RCommands []string `yaml:"rcommands"` + RScripts []string `yaml:"rscripts"` + + // post condition and script + PConditions []string + PScripts []string + + Std string +} diff --git a/libs/update.go b/libs/update.go new file mode 100644 index 0000000..41a7f27 --- /dev/null +++ b/libs/update.go @@ -0,0 +1,28 @@ +package libs + +// Update some config path +type Update struct { + UpdateURL string // url to download the update script + UpdateScript string + MetaDataURL string + UpdateKey string // + UpdateType string // git, http + UpdateConfig string // ~/.osmedeus/update + + UpdateVersion string + UpdateFolder string + UpdateDate string + CleanOldData bool + VulnUpdate bool + GenerateMeta string + ForceUpdate bool + IsUpdateBin bool + EnableUpdate bool + NoUpdate bool +} + +type UpdateMetaData struct { + WorkflowVersion string `json:"workflow_version"` + CoreVersion string `json:"core_version"` + UpdatedAt string `json:"updated_at"` +} diff --git a/libs/version.go b/libs/version.go new file mode 100644 index 0000000..7bb3e2c --- /dev/null +++ b/libs/version.go @@ -0,0 +1,26 @@ +package libs + +import "fmt" + +const ( + // VERSION of this project + VERSION = "v4.6.0" + // DESC description of the tool + DESC = "A Workflow Engine for Offensive Security" + // BINARY name of osmedeus + BINARY = "osmedeus" + // SNAPSHOT binary name of osmedeus + SNAPSHOT = "osm" + // AUTHOR of this + AUTHOR = "@j3ssiejjj" + // DOCS private document + DOCS = "https://docs.osmedeus.org" + // METADATA domain for checking update + METADATA = "https://metadata.osmedeus.org" + // INSTALL default install script + INSTALL = "https://raw.githubusercontent.com/osmedeus/osmedeus-base/master/install.sh" +) + +// TEMP default folder to store inputs +var TEMP = fmt.Sprintf("/tmp/%s-inputs/", SNAPSHOT) +var LDIR = fmt.Sprintf("/tmp/%s-log/", SNAPSHOT) diff --git a/main.go b/main.go new file mode 100644 index 0000000..710da72 --- /dev/null +++ b/main.go @@ -0,0 +1,7 @@ +package main + +import "github.com/j3ssie/osmedeus/cmd" + +func main() { + cmd.Execute() +} diff --git a/provider/action.go b/provider/action.go new file mode 100644 index 0000000..6c24e8a --- /dev/null +++ b/provider/action.go @@ -0,0 +1,201 @@ +package provider + +import ( + "fmt" + "strings" + + "github.com/cenkalti/backoff/v4" + "github.com/fatih/color" + "github.com/j3ssie/osmedeus/utils" + "github.com/spf13/cast" +) + +const ( + GetSSHKey = "get-sshkey" + RunBuild = "run-build" + ListImage = "list-image" + ListInstance = "list-instance" + GetInstanceInfo = "get-instance" + CreateInstance = "create-instance" + BootInstance = "boot-instance" +) + +func (p *Provider) GetSSHKey() (err error) { + if strings.TrimSpace(p.SSHPublicKey) == "" { + return fmt.Errorf("error getting SSHKey -- blank ssh public key") + } + + switch p.ProviderName { + case "do", "digitalocean": + err = p.GetSSHKeyDO() + case "ln", "line", "linode": + err = p.GetSSHKeyLN() + case "aw", "aws", "asw": + err = p.GetSSHKeyAWS() + default: + err = p.GetSSHKeyDO() + } + return err +} + +func (p *Provider) ListInstance() (err error) { + switch p.ProviderName { + case "do", "digitalocean": + err = p.ListInstanceDO() + case "ln", "line", "linode": + err = p.ListInstanceLN() + case "aw", "aws", "asw": + err = p.ListInstanceAWS() + default: + err = p.ListInstanceDO() + } + return err +} + +func (p *Provider) ListSnapShot() (err error) { + switch p.ProviderName { + case "do", "digitalocean": + err = p.ListSnapshotDO() + case "ln", "line", "linode": + err = p.ListSnapshotLN() + case "aw", "aws", "asw": + err = p.ListSnapshotAWS() + default: + err = p.ListSnapshotDO() + } + + if !p.IsBackgroundCheck { + utils.InforF("Found base image snapshot with ID: %v", color.HiCyanString(p.SnapshotID)) + } + return err +} + +func (p *Provider) CreateInstance(name string) (err error) { + var id interface{} + operation := func() error { + switch p.ProviderName { + case "do", "digitalocean": + id, err = p.CreateInstanceDO(name) + if err == nil { + err = p.Action(GetInstanceInfo, id) + } + case "ln", "line", "linode": + id, err = p.CreateInstanceLN(name) + if err == nil { + err = p.Action(BootInstance, id) + err = p.Action(GetInstanceInfo, id) + } + + case "aw", "aws", "asw": + id, err = p.CreateInstanceAWS(name) + if err == nil { + err = p.Action(BootInstance, id) + err = p.Action(GetInstanceInfo, id) + } + default: + id, err = p.CreateInstanceDO(name) + if err == nil { + err = p.Action(GetInstanceInfo, id) + } + } + return err + } + err = backoff.Retry(operation, p.BackOff) + if err != nil { + utils.WarnF("error create instance action %v -- %v", p.ProviderName, name) + return err + } + return nil +} + +// func (p *Provider) CreateInstanceF(name string) (err error) { +// var id int +// switch p.ProviderName { +// case "do", "digitalocean": +// id, err = p.CreateInstanceDO(name) +// if err == nil { +// err = p.Action(GetInstanceInfo, id) +// } +// case "ln", "line", "linode": +// id, err = p.CreateInstanceLN(name) +// if err == nil { +// err = p.Action(BootInstance, id) +// err = p.Action(GetInstanceInfo, id) +// } +// default: +// id, err = p.CreateInstanceDO(name) +// if err == nil { +// err = p.Action(GetInstanceInfo, id) +// } +// } +// return err +// } + +func (p *Provider) BootInstance(id interface{}) (err error) { + switch p.ProviderName { + case "do", "digitalocean": + case "ln", "line", "linode": + err = p.BootInstanceLN(cast.ToInt(id)) + case "aw", "aws", "asw": + // err = p.AllowRootAccessAWS(cast.ToString(id)) + default: + err = p.BootInstanceLN(cast.ToInt(id)) + } + if err != nil { + utils.WarnF("error booting instance: %v", id) + return err + } + return nil +} + +func (p *Provider) GetInstanceInfo(id interface{}) (err error) { + var instance Instance + switch p.ProviderName { + case "do", "digitalocean": + instance, err = p.InstanceInfoDO(cast.ToInt(id)) + case "ln", "line", "linode": + instance, err = p.InstanceInfoLN(cast.ToInt(id)) + case "aw", "aws", "asw": + instance, err = p.InstanceInfoAWS(cast.ToString(id)) + default: + instance, err = p.InstanceInfoDO(cast.ToInt(id)) + } + if err != nil { + utils.WarnF("error getting public IP of instance: %v", color.HiBlueString("%v", id)) + return err + } + p.Instances = append(p.Instances, instance) + return nil +} + +func (p *Provider) DeleteInstance(id string) (err error) { + utils.DebugF("[%v] Delete instance: %v", p.ProviderName, id) + switch p.ProviderName { + case "do", "digitalocean": + err = p.DeleteInstanceDO(id) + case "ln", "line", "linode": + err = p.DeleteInstanceLN(id) + case "aw", "aws", "asw": + err = p.DeleteInstanceAWS(id) + default: + err = p.DeleteInstanceDO(id) + } + return err +} + +func (p *Provider) DeleteOldSnapshot() (err error) { + for _, id := range p.OldSnapShotID { + switch p.ProviderName { + case "do", "digitalocean": + err = p.DeleteSnapShotDO(id) + case "ln", "line", "linode": + err = p.DeleteSnapShotLN(id) + case "aw", "aws", "asw": + err = p.DeleteImageAWS(id) + default: + err = p.DeleteSnapShotDO(id) + } + } + + return err +} diff --git a/provider/building.go b/provider/building.go new file mode 100644 index 0000000..9dbcf00 --- /dev/null +++ b/provider/building.go @@ -0,0 +1,157 @@ +package provider + +import ( + "fmt" + "os" + "path" + "strings" + + "github.com/fatih/color" + "github.com/j3ssie/osmedeus/libs" + "github.com/j3ssie/osmedeus/utils" +) + +func (p *Provider) PrePareBuildData() { + contentFile := path.Join(p.Opt.Env.CloudConfigFolder, fmt.Sprintf("providers/%s.provider", p.ProviderName)) + content := utils.GetFileContent(contentFile) + + data := make(map[string]string) + data["snapshot_name"] = p.SnapshotName + data["api_token"] = p.Token + // for aws only + data["access_key"] = p.AccessKeyId + data["secret_key"] = p.SecretKey + data["source_ami"] = p.ProviderConfig.DefaultImage + + // c.Cloud.ProviderFolder --> ~/.osmedeus/provider/-v4.x-randomstring + p.ProviderConfig.ProviderFolder = path.Join(p.Opt.Env.ProviderFolder, fmt.Sprintf("%s-%s", p.SnapshotName, utils.RandomString(6))) + utils.MakeDir(p.ProviderConfig.ProviderFolder) + data["ProviderFolder"] = p.ProviderConfig.ProviderFolder + + data["image"] = p.ProviderConfig.DefaultImage + data["size"] = p.ProviderConfig.Size + data["region"] = p.ProviderConfig.Region + data["TS"] = utils.GetTS() + + // generate packer content file to run + providerString := utils.RenderText(content, data) + data["Builder"] = providerString + + // ~/osmedeus-base + data["BaseFolder"] = utils.NormalizePath(strings.TrimLeft(p.Opt.Env.BaseFolder, "/")) + data["Plugins"] = p.Opt.Env.BinariesFolder + data["OBin"] = p.Opt.Env.BinariesFolder + data["Data"] = p.Opt.Env.DataFolder + data["Cloud"] = p.Opt.Env.CloudConfigFolder + data["Workflow"] = p.Opt.Env.WorkFlowsFolder + + // ~/.osmedeus/workspaces + data["Workspaces"] = p.Opt.Env.WorkspacesFolder + data["Binary"] = libs.BINARY + data["VERSION"] = libs.VERSION + data["BuildRepo"] = p.Opt.Cloud.BuildRepo + + // for terraform + data["ssh_public_key"] = p.Opt.Cloud.PublicKeyContent + data["root_password"] = fmt.Sprintf("%s-%s", libs.SNAPSHOT, utils.RandomString(8)) + + //spew.Dump("data --> ", data) + //spew.Dump("p.ProviderConfig --> ", p.ProviderConfig) + + p.ProviderConfig.BuildData = data +} + +func (p *Provider) BuildImage() (err error) { + if p.SnapshotFound && !p.Opt.Cloud.ReBuildBaseImage { + return nil + } + + p.PrePareBuildData() + p.DeleteOldSnapshot() + + // p.ProviderConfig.ProviderFolder --> ~/.osmedeus/provider/ + + utils.DebugF("Cleaning old provider build: %s", p.ProviderConfig.ProviderFolder) + os.RemoveAll(p.ProviderConfig.ProviderFolder) + utils.MakeDir(p.ProviderConfig.ProviderFolder) + + // generate provision process + setupContent := utils.GetFileContent(path.Join(p.Opt.Env.CloudConfigFolder, "setup.sh")) + setupContent = utils.RenderText(setupContent, p.ProviderConfig.BuildData) + setupFile := path.Join(p.ProviderConfig.ProviderFolder, "setup.sh") + utils.WriteToFile(setupFile, setupContent) + + // generate build file + var buildContent string + buildContentFile := path.Join(p.Opt.Env.CloudConfigFolder, "general-build.packer") + switch p.ProviderName { + case "do", "digitalocean": + buildContentFile = path.Join(p.Opt.Env.CloudConfigFolder, "digitalocean-build.packer") + if !utils.FileExists(buildContentFile) { + buildContentFile = path.Join(p.Opt.Env.CloudConfigFolder, "do-build.packer") + if !utils.FileExists(buildContentFile) { + buildContentFile = path.Join(p.Opt.Env.CloudConfigFolder, "general-build.packer") + } + } + + case "ln", "line", "linode": + buildContentFile = path.Join(p.Opt.Env.CloudConfigFolder, "linode-build.packer") + if !utils.FileExists(buildContentFile) { + buildContentFile = path.Join(p.Opt.Env.CloudConfigFolder, "ln-build.packer") + } + default: + buildContentFile = path.Join(p.Opt.Env.CloudConfigFolder, "general-build.packer") + } + + buildContent = utils.GetFileContent(buildContentFile) + if buildContent == "" { + errStr := fmt.Sprintf("Build file content not found at: %v", buildContentFile) + utils.ErrorF(errStr) + return fmt.Errorf(errStr) + } + + buildContent = utils.RenderText(buildContent, p.ProviderConfig.BuildData) + buildFile := path.Join(p.ProviderConfig.ProviderFolder, "build.json") + p.ProviderConfig.BuildFile = buildFile + utils.WriteToFile(buildFile, buildContent) + utils.InforF("Write build provision of %s to: %s", color.HiYellowString(p.ProviderName), color.HiCyanString(buildFile)) + + // actually run building + err = p.Action(RunBuild) + if err != nil { + p.SnapshotFound = false + return err + } + + err = p.Action(ListImage) + return err +} + +// RunBuild run the packer command +func (p *Provider) RunBuild() error { + packerBinary := fmt.Sprintf("%s/packer", p.Opt.Env.BinariesFolder) + if !utils.FileExists(packerBinary) { + packerBinary = "packer" + } + + cmd := fmt.Sprintf("%s validate %s", packerBinary, p.ProviderConfig.BuildFile) + out, err := utils.RunCommandWithErr(cmd) + if err != nil { + utils.ErrorF(out) + return err + } + utils.InforF("The Packer file appears to be functioning properly: %s", color.HiCyanString(p.ProviderConfig.BuildFile)) + + // really start to build stuff here + utils.GoodF("Start packer build for: %s", color.HiCyanString(p.ProviderConfig.BuildFile)) + cmd = fmt.Sprintf("%s build %s", packerBinary, p.ProviderConfig.BuildFile) + out, _ = utils.RunCommandWithErr(cmd) + + if !strings.Contains(out, fmt.Sprintf("%v scan -f", libs.BINARY)) { + if !strings.Contains(out, fmt.Sprintf("%v: command not found", libs.BINARY)) { + utils.ErrorF(out) + return fmt.Errorf("error running provisioning") + } + } + return nil +} diff --git a/provider/parser.go b/provider/parser.go new file mode 100644 index 0000000..49bd5de --- /dev/null +++ b/provider/parser.go @@ -0,0 +1,127 @@ +package provider + +import ( + "os" + "math/rand" + "time" + + "github.com/Shopify/yaml" + "github.com/j3ssie/osmedeus/utils" +) + +// ConfigProviders cloud config file +type ConfigProviders struct { + Builder Builder `yaml:"builder"` + Clouds []ConfigProvider `yaml:"clouds"` +} + +// Builder config for builder file +type Builder struct { + BuildRepo string `yaml:"build_repo"` + PublicKey string `yaml:"public_key"` + SecretKey string `yaml:"secret_key"` +} + +// ConfigProvider cloud config file for each provider from ~/osmedeus-base/cloud/provider.yaml +type ConfigProvider struct { + // core part + Name string `yaml:"name"` + Token string `yaml:"token"` + + SecretKey string `yaml:"secret_key"` + AccessKeyId string `yaml:"access_key"` + + Provider string `yaml:"provider"` + DefaultImage string `yaml:"default_image"` + Size string `yaml:"size"` + Region string `yaml:"region"` + Limit int `yaml:"limit"` + Username string `yaml:"username"` + + // BaseImage string `yaml:"base"` + RedactedToken string `yaml:"-"` + Snapshot string `yaml:"-"` + SnapshotID string `yaml:"-"` + InstanceName string `yaml:"-"` + PublicIP string `yaml:"-"` + SshKey string `yaml:"-"` + + // for config + ProviderFolder string `yaml:"-"` + ConfigFile string `yaml:"-"` + BuildFile string `yaml:"-"` + BuildData map[string]string `yaml:"-"` + + // for building + VarsFile string `yaml:"-"` + RunnerFile string `yaml:"-"` + BuildCommand string `yaml:"-"` + BaseFolder string `yaml:"-"` + RawCommand string `yaml:"-"` +} + +// ParseProvider parse cloud file from ~/osmedeus-base/cloud/provider.yaml +func ParseProvider(cloudFile string) (ConfigProviders, error) { + var clouds ConfigProviders + cloudFile = utils.NormalizePath(cloudFile) + + yamlFile, err := os.ReadFile(cloudFile) + if err != nil { + utils.ErrorF("YAML parsing err #%v ", err) + return clouds, err + } + err = yaml.Unmarshal(yamlFile, &clouds) + if err != nil { + utils.ErrorF("Error: %v", err) + return clouds, err + } + + return clouds, nil +} + +// mics function + +const ( + lowerChars = "abcdefghijklmnopqrstuvwxyz" + upperChars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + numberChars = "0123456789" + symbolChars = "!@#$%^&*()+-=[]{}<>?~" +) + +func GeneratePassword(length int) string { + rand.Seed(time.Now().UnixNano()) + + var passwordChars []byte + + // Add at least one of each character type + passwordChars = append(passwordChars, randomChar(lowerChars)) + passwordChars = append(passwordChars, randomChar(upperChars)) + passwordChars = append(passwordChars, randomChar(numberChars)) + passwordChars = append(passwordChars, randomChar(symbolChars)) + + // Add remaining characters randomly + for i := len(passwordChars); i < length; i++ { + charType := rand.Intn(4) // 0 for lower, 1 for upper, 2 for number, 3 for symbol + switch charType { + case 0: + passwordChars = append(passwordChars, randomChar(lowerChars)) + case 1: + passwordChars = append(passwordChars, randomChar(upperChars)) + case 2: + passwordChars = append(passwordChars, randomChar(numberChars)) + case 3: + passwordChars = append(passwordChars, randomChar(symbolChars)) + } + } + + // Shuffle the characters randomly + rand.Shuffle(len(passwordChars), func(i, j int) { + passwordChars[i], passwordChars[j] = passwordChars[j], passwordChars[i] + }) + + return string(passwordChars) +} + +func randomChar(charset string) byte { + return charset[rand.Intn(len(charset))] +} diff --git a/provider/provider.go b/provider/provider.go new file mode 100644 index 0000000..e64f06f --- /dev/null +++ b/provider/provider.go @@ -0,0 +1,265 @@ +package provider + +import ( + "fmt" + "strings" + "time" + + "github.com/cenkalti/backoff/v4" + "github.com/fatih/color" + "github.com/j3ssie/osmedeus/libs" + "github.com/j3ssie/osmedeus/utils" + "github.com/spf13/cast" +) + +type Provider struct { + ProviderName string + Token string + RedactedToken string + // for aws only + AccessKeyId string + SecretKey string + SecurityGroupID string + SecurityGroupName string + + Instances []Instance `json:"-"` + InstanceLimit int + Available bool + HealthCheck bool + + // for create snapshot + SnapshotID string + SnapshotName string + OldSnapShotID []string `json:"-"` + SnapshotFound bool + SSHKeyFound bool + SSHPublicKey string + SSHPrivateKey string + SSHKeyID string + SSHUser string + + // for create + CreatedInstance Instance `json:"-"` + Region string + Size string + SSHKeyName string + + // mics + SwapSizeMap map[string]int `json:"-"` + IsBackgroundCheck bool + + // for building + ProviderConfig ConfigProvider `json:"-"` + Opt libs.Options `json:"-"` + + // for retry + BackOff *backoff.ExponentialBackOff `json:"-"` + // client of vendor + Client interface{} `json:"-"` +} + +type Instance struct { + InstanceID string + InstanceName string + IPAddress string + + // meta data + Region string + Size string + Status string + ImageID string + ImageName string + CPU string + // MB + Memory string + // GB + Disk string + + // more for the osm + InputName string + ProviderName string + CreatedAt string +} + +// InitProvider init provider object to easier interact with cloud provider +func InitProvider(providerName string, token string) (Provider, error) { + var provider Provider + provider.ProviderName = providerName + provider.Token = token + + if providerName == "aws" { + // token should be 'AccessKeyId,SecretKey' + provider.AccessKeyId = strings.TrimSpace(strings.Split(token, ",")[0]) + provider.SecretKey = strings.TrimSpace(strings.Split(token, ",")[1]) + provider.Token = token + } + + provider.InitClient() + return provider, nil +} + +// InitProviderWithConfig init provider object to easier interact with cloud provider +func InitProviderWithConfig(opt libs.Options, providerConfig ConfigProvider) (Provider, error) { + var provider Provider + provider.ProviderName = providerConfig.Provider + provider.Token = providerConfig.Token + + // for aws only + provider.AccessKeyId = providerConfig.AccessKeyId + provider.SecretKey = providerConfig.SecretKey + if provider.AccessKeyId != "" { + provider.Token = provider.AccessKeyId + "," + provider.SecretKey + } + + provider.ProviderConfig = providerConfig + provider.Opt = opt + if opt.Cloud.BackgroundRun { + provider.IsBackgroundCheck = true + } + + if err := provider.InitClient(); err != nil { + return provider, fmt.Errorf("unable to validate token: %v", provider.Token) + } + + provider.Prepare() + return provider, nil +} + +func (p *Provider) InitClient() (err error) { + if p.Token == "" && p.AccessKeyId == "" { + utils.ErrorF("empty or invalid token: %v", p.Token) + return fmt.Errorf("empty or invalid token") + } + if len(p.Token) > 5 { + p.RedactedToken = p.Token[:5] + "***" + p.Token[len(p.Token)-5:len(p.Token)] + if !p.IsBackgroundCheck { + utils.InforF("Init %v provider with token: %v", color.HiYellowString(p.ProviderName), color.HiCyanString(p.RedactedToken)) + } + } + + p.Available = true + switch p.ProviderName { + case "do", "digitalocean": + p.ClientDO() + case "ln", "line", "linode": + p.ClientLinode() + case "aw", "aws", "asw": + p.ClientAWS() + default: + p.ClientDO() + } + + // skip balance check if health check + if p.HealthCheck { + return nil + } + + switch p.ProviderName { + case "do", "digitalocean": + err = p.AccountDO() + case "ln", "line", "linode": + err = p.AccountLN() + case "aw", "aws", "asw": + err = p.AccountAWS() + default: + err = p.AccountDO() + } + + return err +} + +// Prepare setup some default variables +func (p *Provider) Prepare() { + // get snapshot + version := strings.ReplaceAll(strings.TrimSpace(libs.VERSION), " ", "-") + SnapshotName := fmt.Sprintf("%s-base-%s", strings.TrimSpace(libs.SNAPSHOT), version) + p.SnapshotName = SnapshotName + + // sshKey + keyName := fmt.Sprintf("%s-cloud-key", strings.TrimSpace(libs.SNAPSHOT)) + p.SSHKeyName = keyName + + // for retry + b := backoff.NewExponentialBackOff() + // It never stops if MaxElapsedTime == 0. + b.MaxElapsedTime = 1200 * time.Second + b.Multiplier = 2.0 + b.InitialInterval = 30 * time.Second + p.BackOff = b + + if p.Opt.Cloud.Retry > 0 { + b.MaxElapsedTime = time.Duration(p.Opt.Cloud.Retry*60) * time.Second + } + + // setup ssh key + if p.SSHPublicKey == "" { + p.SSHPublicKey = p.Opt.Cloud.PublicKeyContent + } + if p.SSHPrivateKey == "" { + p.SSHPrivateKey = p.Opt.Cloud.SecretKeyContent + } + + utils.DebugF("Get data of cloud provider") + switch p.ProviderName { + case "do", "digitalocean": + p.DefaultDO() + case "ln", "line", "linode": + p.DefaultLinode() + case "aw", "aws", "asw": + p.DefaultAWS() + default: + p.DefaultDO() + } + + p.Action(GetSSHKey) + p.Action(ListImage) + + if p.SSHKeyID != "" { + if !p.IsBackgroundCheck { + utils.InforF("Found SSH Key ID: %v", color.HiBlueString(p.SSHKeyID)) + } + } +} + +func (p *Provider) Action(actionName string, params ...interface{}) error { + var err error + var param interface{} + if len(params) > 0 { + param = params[0] + } + + if !p.IsBackgroundCheck { + utils.InforF("[%v] running action: %v", p.ProviderName, color.HiBlueString(actionName)) + } + + operation := func() error { + switch actionName { + case GetSSHKey: + err = p.GetSSHKey() + case ListInstance: + err = p.ListInstance() + case ListImage: + err = p.ListSnapShot() + case RunBuild: + err = p.RunBuild() + case GetInstanceInfo: + err = p.GetInstanceInfo(param) + case BootInstance: + err = p.BootInstance(param) + case CreateInstance: + err = p.CreateInstance(cast.ToString(param)) + default: + err = p.ListInstance() + } + if err != nil { + utils.ErrorF("error running action %v: %v", color.HiCyanString(actionName), err) + } + return err + } + err = backoff.Retry(operation, p.BackOff) + if err != nil { + utils.ErrorF("error running action %v -- %v", actionName, p.ProviderName) + return err + } + return nil +} diff --git a/provider/provider_aws.go b/provider/provider_aws.go new file mode 100644 index 0000000..94e8f6b --- /dev/null +++ b/provider/provider_aws.go @@ -0,0 +1,516 @@ +package provider + +import ( + "crypto/sha256" + "encoding/base64" + "fmt" + "strings" + "time" + + "github.com/spf13/cast" + "golang.org/x/crypto/ssh" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/costexplorer" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/fatih/color" + "github.com/j3ssie/osmedeus/libs" + "github.com/j3ssie/osmedeus/utils" +) + +// DefaultAWS set some default data for AWS provider +func (p *Provider) DefaultAWS() { + p.Region = "ap-southeast-1" + p.Size = "t2.medium" + p.SecurityGroupName = "osmp-allow-root-access" + p.SSHUser = p.ProviderConfig.Username + + if p.ProviderConfig.Username != "" { + p.SSHUser = "admin" + } + if p.ProviderConfig.Region != "" { + p.Region = p.ProviderConfig.Region + } + if p.ProviderConfig.Size != "" { + p.Size = p.ProviderConfig.Size + } +} + +func (p *Provider) InitSessionAWS() (*session.Session, error) { + sess, err := session.NewSession(&aws.Config{ + Region: aws.String(p.Region), + Credentials: credentials.NewStaticCredentials(p.AccessKeyId, p.SecretKey, ""), + }) + + return sess, err +} + +func (p *Provider) ClientAWS() { + client, err := p.InitSessionAWS() + if err != nil { + panic(err) + } + p.ProviderName = "aws" + p.Client = client +} + +func (p *Provider) ConvertClientAWS() *session.Session { + sess, ok := p.Client.(*session.Session) + if !ok { + utils.ErrorF("error converting aws session %v", ok) + } + sess.Config.Region = aws.String(p.Region) + return sess +} + +func (p *Provider) AccountAWS() error { + ceSvc := costexplorer.New(p.ConvertClientAWS()) + // Set the parameters for the query + now := time.Now() + start := time.Date(now.Year(), now.Month()-1, 1, 0, 0, 0, 0, now.Location()) + end := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, now.Location()) + params := &costexplorer.GetCostAndUsageInput{ + TimePeriod: &costexplorer.DateInterval{ + Start: aws.String(start.Format("2006-01-02")), + End: aws.String(end.Format("2006-01-02")), + }, + Granularity: aws.String(costexplorer.GranularityMonthly), + Metrics: []*string{aws.String("UnblendedCost")}, + } + + // Send the query and get the results + // result, err := ceSvc.GetCostAndUsage(params) + // if err != nil { + // utils.ErrorF("Error getting cost and usage:", err) + // return err + // } + + // Send the query and get the results + result, err := ceSvc.GetCostAndUsage(params) + if err != nil { + utils.ErrorF("Error getting cost and usage: %v", err) + return err + } + + // Print the total cost for the previous month + cost := *result.ResultsByTime[0].Total["UnblendedCost"].Amount + + if !p.IsBackgroundCheck { + utils.InforF("The total cost of AWS services for the this month was %s", color.HiRedString("$"+cost)) + } + return nil +} + +func (p *Provider) GetSSHKeyAWS() error { + ec2Svc := ec2.New(p.ConvertClientAWS()) + // Get a list of all the key pairs in the account + keyPairsOutput, err := ec2Svc.DescribeKeyPairs(&ec2.DescribeKeyPairsInput{}) + if err != nil { + utils.ErrorF("Error describing key pairs: %v", err) + return err + } + + pubKeyBytes := []byte(p.SSHPublicKey) + // Parse the key, other info ignored + pubKey, _, _, _, err := ssh.ParseAuthorizedKey(pubKeyBytes) + if err != nil { + utils.ErrorF("%v", err) + return err + } + hash := sha256.Sum256(pubKey.Marshal()) + sshHash := base64.StdEncoding.EncodeToString(hash[:]) + + // Check if your SSH key is present in the list + for _, keyPair := range keyPairsOutput.KeyPairs { + // found the same key name but different key fingerprint + if *keyPair.KeyName == p.SSHKeyName && *keyPair.KeyFingerprint != sshHash { + // Delete the key pair + _, err := ec2Svc.DeleteKeyPair(&ec2.DeleteKeyPairInput{ + KeyName: aws.String(*keyPair.KeyName), + }) + if err != nil { + utils.ErrorF("%v", err) + } + utils.InforF("Successfully deleted key pair %s", color.HiBlueString(*keyPair.KeyName)) + } + + if *keyPair.KeyFingerprint == sshHash { + p.SSHKeyID = cast.ToString(*keyPair.KeyPairId) + p.SSHKeyFound = true + break + } + + } + + if p.SSHKeyFound { + utils.DebugF("Your SSH key was found in the account: %v -- %v", color.HiCyanString(p.SSHKeyName), color.HiCyanString(p.SSHKeyID)) + return nil + } + + // Import the SSH key into your AWS account + result, err := ec2Svc.ImportKeyPair(&ec2.ImportKeyPairInput{ + KeyName: aws.String(p.SSHKeyName), + PublicKeyMaterial: []byte(p.SSHPublicKey), + }) + if err != nil { + utils.ErrorF("Error create key pairs: %v", err) + return err + } + utils.DebugF("Successfully imported SSH key: %v -- %v", color.HiCyanString(*result.KeyName), color.HiCyanString(*result.KeyPairId)) + p.SSHKeyID = cast.ToString(*result.KeyPairId) + p.SSHKeyFound = true + + return nil +} + +func (p *Provider) ListSnapshotAWS() error { + svc := ec2.New(p.ConvertClientAWS()) + + // listing only image that own by you + self := "self" + ownImages := &ec2.DescribeImagesInput{Owners: []*string{&self}} + result, err := svc.DescribeImages(ownImages) + if err != nil { + utils.ErrorF("err: Unable to list images, %v", err) + return err + } + + for _, item := range result.Images { + if strings.HasPrefix(*item.Name, libs.SNAPSHOT) { + p.OldSnapShotID = append(p.OldSnapShotID, *item.ImageId) + } + + if strings.TrimSpace(*item.Name) == strings.TrimSpace(p.SnapshotName) { + utils.DebugF("Found base image snapshot with ID: %s", color.HiBlueString(*item.ImageId)) + p.SnapshotID = *item.ImageId + p.SnapshotName = *item.Name + p.SnapshotFound = true + } + } + + return nil +} + +func (p *Provider) DeleteImageAWS(id string) error { + if p.SnapshotID == "" { + return nil + } + svc := ec2.New(p.ConvertClientAWS()) + deletedImage := &ec2.DeregisterImageInput{ImageId: &p.SnapshotID} + _, err := svc.DeregisterImage(deletedImage) + if err != nil { + utils.ErrorF("err: Unable to delete snapshot: %v -- %v", id, err) + return err + } + utils.InforF("Deleted image ID: %v", color.HiRedString(p.SnapshotID)) + p.DeleteSnapshotAWS() + return nil +} + +func (p *Provider) DeleteSnapshotAWS() error { + svc := ec2.New(p.ConvertClientAWS()) + + // List all snapshots + result, err := svc.DescribeSnapshots(&ec2.DescribeSnapshotsInput{ + OwnerIds: []*string{aws.String("self")}, + MaxResults: aws.Int64(1000), + }) + if err != nil { + utils.ErrorF("Error listing snapshots: %v", err) + return err + } + + var snapshotIDs []string + for _, snapshot := range result.Snapshots { + + for _, tag := range snapshot.Tags { + if *tag.Key == "Name" && *tag.Value == "Osmedeus Premium Image" { + snapshotIDs = append(snapshotIDs, *snapshot.SnapshotId) + } + } + } + + for _, snapshotID := range snapshotIDs { + _, err = svc.DeleteSnapshot(&ec2.DeleteSnapshotInput{ + SnapshotId: aws.String(snapshotID), + }) + if err != nil { + utils.ErrorF("Error deleting snapshot: %v -- %v", snapshotID, err) + } else { + utils.DebugF("Delted snapshot ID: %v", color.HiRedString(snapshotID)) + } + } + + return nil +} + +func (p *Provider) ListInstanceAWS() error { + svc := ec2.New(p.ConvertClientAWS()) + result, err := svc.DescribeInstances(nil) + if err != nil { + utils.ErrorF("err: Unable to list ec2 instances: %v", err) + return err + } + + var numberOfInstance int + for i := range result.Reservations { + for _, instance := range result.Reservations[i].Instances { + if *instance.State.Name != "running" { + continue + } + numberOfInstance += 1 + + launchTime := *instance.LaunchTime + creationDate := launchTime.Format(time.RFC1123) + parsedInstance := Instance{ + InstanceID: cast.ToString(*instance.InstanceId), + IPAddress: *instance.PublicIpAddress, + InstanceName: *instance.State.Name, + ImageID: cast.ToString(*instance.ImageId), + // ImageName: instance.Image.Name, + // Region: instance.Region.Slug, + // Region: *instance.Architecture, + // Memory: cast.ToString(instance.Memory), + // CPU: cast.ToString(instance.Vcpus), + // Disk: cast.ToString(instance.Disk), + Status: *instance.State.Name, + CreatedAt: cast.ToString(creationDate), + InputName: "", + ProviderName: "aws", + } + + p.Instances = append(p.Instances, parsedInstance) + } + } + + utils.InforF("Found %v running instances", color.HiMagentaString("%v", numberOfInstance)) + // check if we reach max instance number + if p.InstanceLimit > 0 { + if len(p.Instances) >= p.InstanceLimit { + p.Available = false + } + } + + return nil +} + +func (p *Provider) CreateInstanceAWS(InstanceName string) (instanctID string, err error) { + svc := ec2.New(p.ConvertClientAWS()) + p.CreateSecurityGroup() + + // Set the parameters for the instance + params := &ec2.RunInstancesInput{ + // InstanceName: aws.String(name), + ImageId: aws.String(p.SnapshotID), // Replace with the ID of the image you want to use + InstanceType: aws.String(p.Size), // Specify the instance type like t2.micro + MinCount: aws.Int64(1), + MaxCount: aws.Int64(1), + KeyName: aws.String(p.SSHKeyName), + SecurityGroups: []*string{ + aws.String(p.SecurityGroupName), + }, + TagSpecifications: []*ec2.TagSpecification{ + { + ResourceType: aws.String("instance"), + Tags: []*ec2.Tag{ + { + Key: aws.String("Name"), + Value: aws.String(InstanceName), + }, + }, + }, + }, + } + + // Create the instance + result, err := svc.RunInstances(params) + if err != nil { + utils.ErrorF("Error creating instance: %v", err) + return + } + + // Get the instance ID + instanctID = *result.Instances[0].InstanceId + utils.InforF("Successfully Created Instance ID: %v -- %v", color.HiBlueString(instanctID), color.HiBlueString(InstanceName)) + utils.DebugF("Waiting for the instance %v to be ready...", color.HiBlueString(instanctID)) + + time.Sleep(60 * time.Second) + // Get the instance state + for i := 0; i < 10; i++ { + if p.InstanceReady(instanctID) == nil { + return instanctID, nil + } + time.Sleep(60 * time.Second) + } + + return instanctID, nil +} + +func (p *Provider) InstanceReady(instanceID string) error { + svc := ec2.New(p.ConvertClientAWS()) + + // Describe the instance + params := &ec2.DescribeInstancesInput{ + InstanceIds: []*string{ + aws.String(instanceID), + }, + } + _, err := svc.DescribeInstances(params) + if err != nil { + return err + } + + return nil +} + +func (p *Provider) InstanceReboot(instanceID string) error { + svc := ec2.New(p.ConvertClientAWS()) + + // Create the input for the RebootInstances operation + params := &ec2.RebootInstancesInput{ + InstanceIds: []*string{aws.String(instanceID)}, + } + + // Call the RebootInstances operation + _, err := svc.RebootInstances(params) + if err != nil { + fmt.Println("Error rebooting instance:", err) + return err + } + return nil +} + +func (p *Provider) DeleteInstanceAWS(id string) error { + svc := ec2.New(p.ConvertClientAWS()) + + // Set the parameters for the instance + params := &ec2.TerminateInstancesInput{ + InstanceIds: []*string{ + aws.String(id), // Replace with the ID of the instance you want to delete + }, + } + + // Delete the instance + _, err := svc.TerminateInstances(params) + if err != nil { + utils.ErrorF("Error deleting instance: %v", err) + return err + } + + utils.InforF("Successfully Deleted instance ID: %v", color.HiRedString(id)) + return nil +} + +func (p *Provider) InstanceInfoAWS(id string) (Instance, error) { + var parsedInstance Instance + + svc := ec2.New(p.ConvertClientAWS()) + + // Set the parameters for the instance + params := &ec2.DescribeInstancesInput{ + InstanceIds: []*string{ + aws.String(id), + }, + } + + // Get the instance information + result, err := svc.DescribeInstances(params) + if err != nil { + utils.ErrorF("Error getting instance information: %v", err) + return parsedInstance, err + } + + // Print the instance information + for _, reservation := range result.Reservations { + for _, instance := range reservation.Instances { + + launchTime := *instance.LaunchTime + creationDate := launchTime.Format(time.RFC1123) + + var instanceName string + for _, tag := range instance.Tags { + if *tag.Key == "Name" { + instanceName = *tag.Value + break + } + } + + parsedInstance = Instance{ + InstanceID: cast.ToString(*instance.InstanceId), + IPAddress: *instance.PublicIpAddress, + InstanceName: instanceName, + ImageID: cast.ToString(*instance.ImageId), + Status: *instance.State.Name, + CreatedAt: cast.ToString(creationDate), + InputName: "", + ProviderName: "aws", + } + + } + } + + p.CreatedInstance = parsedInstance + utils.DebugF("Instance ID Info: %v -- %v -- %v", color.HiBlueString(p.CreatedInstance.InstanceID), p.CreatedInstance.InstanceName, p.CreatedInstance.IPAddress) + return parsedInstance, nil +} + +func (p *Provider) CreateSecurityGroup() error { + svc := ec2.New(p.ConvertClientAWS()) + + // Set the parameters for the security group + params := &ec2.DescribeSecurityGroupsInput{ + GroupNames: []*string{ + aws.String(p.SecurityGroupName), // Replace with the ID of the security group you want to check + }, + } + + // Get the security group information + scGroups, err := svc.DescribeSecurityGroups(params) + if err == nil { + // Print the security group information + for _, group := range scGroups.SecurityGroups { + if *group.GroupName == p.SecurityGroupName { + p.SecurityGroupID = *group.GroupId + utils.DebugF("Security Group allow root access has been found: %v", color.HiBlueString(p.SecurityGroupID)) + return nil + } + } + } + + // only create if not found + + // Create the security group + result, err := svc.CreateSecurityGroup(&ec2.CreateSecurityGroupInput{ + GroupName: aws.String("osmp-allow-root-access"), + Description: aws.String("Security group for allowing root access to EC2 instances"), + }) + if err != nil { + utils.ErrorF("Error creating security group: %v", err) + return err + } + + // Add a rule to the security group to allow SSH access from any IP address + _, err = svc.AuthorizeSecurityGroupIngress(&ec2.AuthorizeSecurityGroupIngressInput{ + GroupId: aws.String(*result.GroupId), + IpPermissions: []*ec2.IpPermission{ + { + FromPort: aws.Int64(22), + ToPort: aws.Int64(22), + IpProtocol: aws.String("tcp"), + IpRanges: []*ec2.IpRange{ + {CidrIp: aws.String("0.0.0.0/0")}, + }, + }, + }, + }) + if err != nil { + utils.ErrorF("Error adding rule to security group: %v", err) + return err + } + + p.SecurityGroupID = *result.GroupId + utils.DebugF("Security Group allow root access has been found: %v", color.HiBlueString(p.SecurityGroupID)) + return nil +} diff --git a/provider/provider_digitalocean.go b/provider/provider_digitalocean.go new file mode 100644 index 0000000..a001c90 --- /dev/null +++ b/provider/provider_digitalocean.go @@ -0,0 +1,294 @@ +package provider + +import ( + "context" + "fmt" + "io" + "strings" + "time" + + "github.com/digitalocean/godo" + "github.com/fatih/color" + "github.com/j3ssie/osmedeus/libs" + "github.com/j3ssie/osmedeus/utils" + "github.com/spf13/cast" +) + +// DefaultDO set some default data for DO provider +func (p *Provider) DefaultDO() { + p.Region = "sfo3" + p.Size = "s-2vcpu-4gb" + p.SSHUser = p.ProviderConfig.Username + + if p.ProviderConfig.Username != "" { + p.SSHUser = "root" + } + if p.ProviderConfig.Region != "" { + p.Region = p.ProviderConfig.Region + } + if p.ProviderConfig.Size != "" { + p.Size = p.ProviderConfig.Size + } +} + +func (p *Provider) ClientDO() { + client := godo.NewFromToken(p.Token) + p.Client = client +} + +func (p *Provider) ConvertClientDO() *godo.Client { + client, ok := p.Client.(*godo.Client) + if !ok { + utils.ErrorF("error converting digital ocean session %v", ok) + } + return client +} + +func (p *Provider) AccountDO() error { + client := p.ConvertClientDO() + ctx := context.TODO() + + account, _, err := client.Account.Get(ctx) + if err != nil { + return fmt.Errorf("error getting account information") + } + p.InstanceLimit = account.DropletLimit + + bill, _, err := client.Balance.Get(ctx) + if err != nil { + return fmt.Errorf("error getting account information") + } + + if !p.IsBackgroundCheck { + utils.InforF("Account Billing Information: MonthToDateBalance: %v -- AccountBalance: %v", strings.TrimLeft(color.HiRedString(bill.MonthToDateBalance), "-"), color.HiGreenString(bill.AccountBalance)) + } + + return nil +} + +func (p *Provider) ListInstanceDO() error { + client := p.ConvertClientDO() + + ctx := context.TODO() + opt := &godo.ListOptions{ + Page: 1, + PerPage: 1000, + } + + droplets, _, err := client.Droplets.List(ctx, opt) + if err != nil { + return fmt.Errorf("error getting digital ocean instance") + } + utils.DebugF("found %v instances", len(droplets)) + + for _, instance := range droplets { + ipAddress, ok := instance.PublicIPv4() + if ok != nil || ipAddress == "" { + utils.ErrorF("Instance has no public IP: %v -- %v", instance.ID, instance.Name) + continue + } + parsedInstance := Instance{ + InstanceID: cast.ToString(instance.ID), + IPAddress: ipAddress, + InstanceName: instance.Name, + ImageID: cast.ToString(instance.Image.ID), + ImageName: instance.Image.Name, + Region: instance.Region.Slug, + Memory: cast.ToString(instance.Memory), + CPU: cast.ToString(instance.Vcpus), + Disk: cast.ToString(instance.Disk), + Status: instance.Status, + CreatedAt: instance.Created, + + InputName: "", + ProviderName: "do", + } + + p.Instances = append(p.Instances, parsedInstance) + } + + // check if we reach max instance number + if p.InstanceLimit > 0 { + if len(p.Instances) >= p.InstanceLimit { + p.Available = false + } + } + + return nil +} + +func (p *Provider) GetSSHKeyDO() error { + client := p.ConvertClientDO() + + ctx := context.TODO() + opt := &godo.ListOptions{ + Page: 1, + PerPage: 1000, + } + + keys, _, err := client.Keys.List(ctx, opt) + if err != nil { + return fmt.Errorf("error listing ssh key -- %v", err) + } + + for _, key := range keys { + if strings.TrimSpace(key.PublicKey) == strings.TrimSpace(p.SSHPublicKey) { + p.SSHKeyID = cast.ToString(key.ID) + p.SSHKeyFound = true + utils.DebugF("Found SSH Key: %v -- %v ", color.HiCyanString(key.Name), color.HiCyanString(p.SSHKeyID)) + } + } + + // create one if not found + if !p.SSHKeyFound { + utils.DebugF("No SSHKey found. create a new one") + createRequest := &godo.KeyCreateRequest{ + Name: p.SSHKeyName, + PublicKey: p.SSHPublicKey, + } + transfer, _, err := client.Keys.Create(ctx, createRequest) + if err == nil { + p.SSHKeyID = cast.ToString(transfer.ID) + p.SSHKeyFound = true + utils.DebugF("Created new SSH Key: %v", color.HiCyanString(p.SSHKeyID)) + } else { + return fmt.Errorf("error create ssh key -- %v", err) + } + } + + return nil +} + +func (p *Provider) ListSnapshotDO() error { + client := p.ConvertClientDO() + + ctx := context.TODO() + opt := &godo.ListOptions{ + Page: 1, + PerPage: 1000, + } + + snapshots, _, err := client.Snapshots.List(ctx, opt) + if err != nil { + return fmt.Errorf("error getting digital ocean snapshot") + } + for _, instance := range snapshots { + name := instance.Name + id := cast.ToString(instance.ID) + + if strings.HasPrefix(name, libs.SNAPSHOT) { + p.OldSnapShotID = append(p.OldSnapShotID, id) + } + + if strings.TrimSpace(name) == strings.TrimSpace(p.SnapshotName) { + utils.DebugF("Found base image snapshot with ID: %s", color.HiBlueString(id)) + p.SnapshotID = id + p.SnapshotName = name + p.SnapshotFound = true + + } + } + + return nil +} + +func (p *Provider) CreateInstanceDO(name string) (dropletId int, err error) { + client := p.ConvertClientDO() + + ctx := context.TODO() + createRequest := &godo.DropletCreateRequest{ + Name: name, + Region: p.Region, + Size: p.Size, + Image: godo.DropletCreateImage{ + ID: cast.ToInt(p.SnapshotID), + //Slug: "ubuntu-16-04-x64", + }, + // SSHKeys: []godo.DropletCreateSSHKey{ + // godo.DropletCreateSSHKey{ID: cast.ToInt(p.SSHKeyID)}, + // }, + // Tags: []string{libs.SNAPSHOT}, + + SSHKeys: []godo.DropletCreateSSHKey{ + {ID: cast.ToInt(p.SSHKeyID)}, + }, + Tags: []string{libs.SNAPSHOT}, + } + + instance, res, err := client.Droplets.Create(ctx, createRequest) + if err != nil { + utils.ErrorF("error create digital ocean instance -- %v", err) + content, ok := io.ReadAll(res.Body) + if ok == nil { + fmt.Println(string(content)) + } + return dropletId, fmt.Errorf("error create digital ocean instance -- %v", err) + } + + // get droplet IP info + dropletId = instance.ID + utils.DebugF("Created instance %v", color.HiBlueString("%v", instance.ID)) + utils.DebugF("Waiting for the instance %v to be ready...", color.HiBlueString("%v", instance.ID)) + time.Sleep(100 * time.Second) + return dropletId, nil +} + +func (p *Provider) InstanceInfoDO(id int) (Instance, error) { + var parsedInstance Instance + client := p.ConvertClientDO() + ctx := context.TODO() + instance, _, err := client.Droplets.Get(ctx, id) + if err != nil { + return parsedInstance, fmt.Errorf("error get instance info:") + } + + ipAddress, err := instance.PublicIPv4() + if err != nil || ipAddress == "" { + return parsedInstance, fmt.Errorf("no public ip address yet") + } + parsedInstance = Instance{ + InstanceID: cast.ToString(instance.ID), + IPAddress: ipAddress, + InstanceName: instance.Name, + ImageID: cast.ToString(instance.Image.ID), + ImageName: instance.Image.Name, + Region: instance.Region.Slug, + Memory: cast.ToString(instance.Memory), + CPU: cast.ToString(instance.Vcpus), + Disk: cast.ToString(instance.Disk), + Status: instance.Status, + CreatedAt: instance.Created, + + InputName: "", + ProviderName: "do", + } + p.CreatedInstance = parsedInstance + utils.DebugF("Successfully Created Instance: %v -- %v -- %v", p.CreatedInstance.InstanceID, p.CreatedInstance.InstanceName, p.CreatedInstance.IPAddress) + + return parsedInstance, nil +} + +func (p *Provider) DeleteInstanceDO(id string) error { + client := p.ConvertClientDO() + ctx := context.TODO() + _, err := client.Droplets.Delete(ctx, cast.ToInt(id)) + if err != nil { + utils.ErrorF("error delete instance -- %v", err) + return fmt.Errorf("error delete instance") + } + utils.InforF("Successfully Deleted instance ID: %v", color.HiRedString(id)) + return nil +} + +func (p *Provider) DeleteSnapShotDO(id string) error { + client := p.ConvertClientDO() + ctx := context.TODO() + + _, err := client.Snapshots.Delete(ctx, id) + if err != nil { + utils.ErrorF("error delete snapshot -- %v", err) + return fmt.Errorf("error delete instance") + } + utils.InforF("Deleted snapshot ID: %v", color.HiRedString(id)) + return nil +} diff --git a/provider/provider_linode.go b/provider/provider_linode.go new file mode 100644 index 0000000..1cb7cae --- /dev/null +++ b/provider/provider_linode.go @@ -0,0 +1,477 @@ +package provider + +import ( + "context" + "encoding/json" + "fmt" + "log" + "net/http" + "strings" + "time" + + "github.com/fatih/color" + "github.com/j3ssie/osmedeus/libs" + "github.com/j3ssie/osmedeus/utils" + "github.com/linode/linodego" + "github.com/spf13/cast" + "golang.org/x/oauth2" +) + +// DefaultLinode set some default data for DO provider +func (p *Provider) DefaultLinode() { + p.Region = "us-east" + p.Size = "g6-standard-1" + p.SSHUser = p.ProviderConfig.Username + + if p.ProviderConfig.Username != "" { + p.SSHUser = "root" + } + if p.ProviderConfig.Region != "" { + p.Region = p.ProviderConfig.Region + } + if p.ProviderConfig.Size != "" { + p.Size = p.ProviderConfig.Size + } + + if p.Opt.Cloud.Size != "" { + p.Size = p.Opt.Cloud.Size + } + if p.Opt.Cloud.Region != "" { + p.Region = p.Opt.Cloud.Region + } + + p.LinodeDiskMap() +} + +func (p *Provider) ClientLinode() error { + tokenSource := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: p.Token}) + oauth2Client := &http.Client{ + Transport: &oauth2.Transport{ + Source: tokenSource, + }, + } + + linodeClient := linodego.NewClient(oauth2Client) + + p.Client = linodeClient + return nil +} + +func (p *Provider) ConvertClientLinode() linodego.Client { + client, ok := p.Client.(linodego.Client) + if !ok { + utils.ErrorF("error converting linode session %v", ok) + } + return client +} + +func (p *Provider) AccountLN() error { + client := p.ConvertClientLinode() + ctx := context.TODO() + account, err := client.GetAccount(ctx) + if err != nil { + return fmt.Errorf("error getting account information") + } + if !p.IsBackgroundCheck { + utils.InforF("Account Billing Information: BalanceUninvoiced: %v -- AccountBalance: %v", color.HiRedString("%v", account.BalanceUninvoiced), color.HiGreenString("%v", account.Balance)) + } + helper := false + opt := linodego.AccountSettingsUpdateOptions{ + NetworkHelper: &helper, + } + + // @TODO: no idea why this function false + // client.UpdateAccountSettings(context.Background(), opt) + + req := client.R(ctx).SetResult(&linodego.AccountSettings{}) + if bodyData, err := json.Marshal(&opt); err == nil { + req.URL = "https://api.linode.com/v4/account/settings" + //fmt.Println("err marshal", err) + body := string(bodyData) + req.SetBody(body).Put(req.URL) + } + //} + return err +} + +// LinodeTest list all instances +func (p *Provider) LinodeTest() error { + linodeClient := p.ConvertClientLinode() + + res, err := linodeClient.GetInstance(context.Background(), 4090913) + if err != nil { + log.Fatal(err) + } + fmt.Printf("%v", res) + + return nil +} + +func (p *Provider) ListInstanceLN() error { + client := p.ConvertClientLinode() + + opt := &linodego.ListOptions{ + PageOptions: nil, + Filter: "", + } + + instances, err := client.ListInstances(context.TODO(), opt) + if err != nil { + return fmt.Errorf("error getting linode instance") + } + + for _, instance := range instances { + //instance.IPv4 + if len(instance.IPv4) == 0 { + utils.ErrorF("Instance has no public IP: %v -- %v", instance.ID, instance.Label) + continue + } + ipAddress := instance.IPv4[0] + + parsedInstance := Instance{ + InstanceID: cast.ToString(instance.ID), + IPAddress: cast.ToString(ipAddress), + InstanceName: instance.Label, + ImageID: cast.ToString(instance.Image), + ImageName: instance.Image, + Region: instance.Region, + Memory: cast.ToString(instance.Specs.Memory), + CPU: cast.ToString(instance.Specs.VCPUs), + Disk: cast.ToString(instance.Specs.Disk), + Status: cast.ToString(instance.Status), + CreatedAt: cast.ToString(instance.Created), + + InputName: "", + ProviderName: "do", + } + + p.Instances = append(p.Instances, parsedInstance) + } + return nil +} + +func (p *Provider) GetSSHKeyLN() error { + client := p.ConvertClientLinode() + ctx := context.TODO() + opt := &linodego.ListOptions{ + PageOptions: nil, + Filter: "", + } + + keys, err := client.ListSSHKeys(ctx, opt) + if err != nil { + return fmt.Errorf("error listing ssh key -- %v", err) + } + + for _, key := range keys { + if strings.TrimSpace(key.SSHKey) == strings.TrimSpace(p.SSHPublicKey) { + p.SSHKeyID = cast.ToString(key.ID) + p.SSHKeyFound = true + utils.DebugF("Found SSH Key: %v -- %v ", key.Label, p.SSHKeyID) + } + } + + // create one if not found + if !p.SSHKeyFound { + utils.DebugF("No SSHKey found. create a new one") + createRequest := linodego.SSHKeyCreateOptions{ + Label: p.SSHKeyName, + SSHKey: p.SSHPublicKey, + } + key, err := client.CreateSSHKey(ctx, createRequest) + if err == nil { + p.SSHKeyID = cast.ToString(key.ID) + p.SSHKeyFound = true + utils.DebugF("Created new SSH Key: %v", p.SSHKeyID) + } + } + + return nil +} + +func (p *Provider) ListSnapshotLN() error { + client := p.ConvertClientLinode() + ctx := context.TODO() + opt := &linodego.ListOptions{ + PageOptions: nil, + Filter: "", + } + + snapshots, err := client.ListImages(ctx, opt) + if err != nil { + return fmt.Errorf("error getting linode images") + } + for _, image := range snapshots { + name := image.Label + id := cast.ToString(image.ID) + if strings.HasPrefix(name, libs.SNAPSHOT) { + p.OldSnapShotID = append(p.OldSnapShotID, id) + } + + if strings.TrimSpace(name) == strings.TrimSpace(p.SnapshotName) { + utils.DebugF("Found base image snapshot with ID: %s", id) + p.SnapshotID = id + p.SnapshotName = name + p.SnapshotFound = true + } + } + + return nil +} + +func (p *Provider) LinodeDiskMap() { + p.SwapSizeMap = make(map[string]int) + p.SwapSizeMap["g6-nanode-1"] = 20000 + p.SwapSizeMap["g6-standard-1"] = 4000 + p.SwapSizeMap["g6-standard-2"] = 8000 + p.SwapSizeMap["g6-standard-4"] = 16000 + p.SwapSizeMap["g6-standard-6"] = 32000 +} + +func (p *Provider) CreateInstanceLN(name string) (dropletId int, err error) { + client := p.ConvertClientLinode() + + ctx := context.TODO() + booted := false + swapSize := 4000 + if swap, ok := p.SwapSizeMap[p.Size]; ok { + swapSize = swap + } + + createRequest := linodego.InstanceCreateOptions{ + Region: p.Region, + Type: p.Size, + Label: name, + // RootPass: utils.RandomString(10), + RootPass: GeneratePassword(16), + AuthorizedKeys: []string{ + p.SSHPublicKey, + }, + //AuthorizedUsers: []string{"root"}, + Image: p.SnapshotID, + Tags: []string{libs.SNAPSHOT}, + + SwapSize: &swapSize, + Booted: &booted, + //StackScriptID: 0, + //Group: "", + //StackScriptData: nil, + //BackupID: 0, + //BackupsEnabled: false, + //PrivateIP: false, + } + + utils.DebugF("Creating instance based on %v/%v image with password: %v", p.Size, p.SnapshotID, createRequest.RootPass) + instance, err := client.CreateInstance(ctx, createRequest) + if err != nil { + utils.ErrorF("error create linode instance %v -- %v", name, err) + return dropletId, fmt.Errorf("error create linode instance -- %v", err) + } + + //spew.Dump(instance) + // get droplet IP info + dropletId = instance.ID + time.Sleep(60 * time.Second) + return dropletId, nil +} + +func (p *Provider) BootInstanceLN(dropletId int) error { + client := p.ConvertClientLinode() + + ctx := context.TODO() + err := client.BootInstance(ctx, dropletId, 0) + return err +} + +func (p *Provider) MountDiskLN(dropletId int) error { + client := p.ConvertClientLinode() + utils.InforF("Mounting disk: %v", dropletId) + ctx := context.TODO() + + disk, err := client.CreateInstanceDisk(ctx, dropletId, linodego.InstanceDiskCreateOptions{ + Label: "test1", + Filesystem: "ext4", + Size: 2000, + }) + if err != nil { + utils.ErrorF("Error creating disk for resize: %s", err) + } + + disk, err = client.WaitForInstanceDiskStatus(ctx, dropletId, disk.ID, linodego.DiskReady, 180) + if err != nil { + utils.ErrorF("Error waiting for disk readiness for resize: %s", err) + return err + } + err = client.ResizeInstanceDisk(ctx, dropletId, disk.ID, 4000) + if err != nil { + utils.ErrorF("Error resizing instance disk: %s", err) + } + + return nil +} + +func (p *Provider) InstanceInfoLN(id int) (Instance, error) { + var parsedInstance Instance + client := p.ConvertClientLinode() + + instance, err := client.GetInstance(context.TODO(), id) + if err != nil { + return parsedInstance, fmt.Errorf("error getting linode instance") + } + + if len(instance.IPv4) == 0 { + utils.ErrorF("Instance has no public IP: %v -- %v", instance.ID, instance.Label) + return parsedInstance, fmt.Errorf("no public ip address yet") + } + ipAddress := instance.IPv4[0] + parsedInstance = Instance{ + InstanceID: cast.ToString(instance.ID), + IPAddress: cast.ToString(ipAddress), + InstanceName: instance.Label, + ImageID: cast.ToString(instance.Image), + ImageName: instance.Image, + Region: instance.Region, + Memory: cast.ToString(instance.Specs.Memory), + CPU: cast.ToString(instance.Specs.VCPUs), + Disk: cast.ToString(instance.Specs.Disk), + Status: cast.ToString(instance.Status), + CreatedAt: cast.ToString(instance.Created), + InputName: "", + ProviderName: "linode", + } + p.CreatedInstance = parsedInstance + utils.DebugF("Created instance ID: %v -- %v -- %v", p.CreatedInstance.InstanceID, p.CreatedInstance.InstanceName, p.CreatedInstance.IPAddress) + + return parsedInstance, nil +} + +func (p *Provider) DeleteInstanceLN(id string) error { + client := p.ConvertClientLinode() + + ctx := context.TODO() + err := client.DeleteInstance(ctx, cast.ToInt(id)) + if err != nil { + utils.ErrorF("error delete instance -- %v", err) + return fmt.Errorf("error delete instance") + } + utils.InforF("Deleted instance ID: %v", color.HiRedString(id)) + return nil +} + +func (p *Provider) DeleteSnapShotLN(id string) error { + client := p.ConvertClientLinode() + + ctx := context.TODO() + err := client.DeleteImage(ctx, id) + if err != nil { + utils.ErrorF("error delete snapshot -- %v", err) + return fmt.Errorf("error delete instance") + } + utils.InforF("Deleted snapshot ID: %v", color.HiRedString(id)) + return nil +} + +// @NOTE: old note for increase disk in linode +//func (c *CloudRunner) MountDiskLinode() error { +// diskSize := p.DisksMap[c.Cloud.Size] +// if diskSize == "" { +// return fmt.Errorf("can't found disk size") +// } +// +// utils.InforF("Mounting Disk size %s for instance %s", diskSize, c.InstanceID) +// cmd := c.Prefix + fmt.Sprintf(`linodes disks-list %s`, c.InstanceID) +// out := c.RetryCommandWithExcludeString(cmd, `Request failed:`) +// jsonParsed, err := gabs.ParseJSON([]byte(out)) +// if err != nil { +// utils.ErrorF("error when parsing content of droplet list") +// return err +// } +// +// var diskID string +// for _, instance := range jsonParsed.Children() { +// filesystem := cast.ToString(instance.S("filesystem").Data()) +// if strings.HasPrefix(filesystem, "ext") { +// diskID = cast.ToString(instance.S("id").Data()) +// } +// } +// if diskID == "" { +// return fmt.Errorf("error to find disk ID") +// } +// +// cmd = c.Prefix + fmt.Sprintf(`linodes disk-resize %s %s --size %s`, c.InstanceID, diskID, diskSize) +// out = c.RetryCommandWithExcludeString(cmd, `Request failed:`) +// if strings.Contains(out, "Request failed") { +// return fmt.Errorf("error to mount disk") +// } +// +// time.Sleep(5 * time.Second) +// // everything done booting the instance up +// cmd = c.Prefix + fmt.Sprintf(`linodes boot %s `, c.InstanceID) +// out = c.RetryCommandWithExcludeString(cmd, `Request failed:`) +// if strings.Contains(out, "Request failed") { +// return fmt.Errorf("error to mount disk") +// } +// +// return nil +//} +// +//func (c *CloudRunner) LinodeDiskMap() { +// c.DisksMap = make(map[string]string) +// c.DisksMap["g6-nanode-1"] = "20000" +// c.DisksMap["g6-standard-1"] = "45000" +// c.DisksMap["g6-standard-2"] = "75000" +// c.DisksMap["g6-standard-4"] = "150000" +// c.DisksMap["g6-standard-6"] = "320000" +//} +// +//var reservedAddrRanges []*net.IPNet +// +//var ReservedCIDRs = []string{ +// "192.168.0.0/16", +// "172.16.0.0/12", +// "10.0.0.0/8", +// "127.0.0.0/8", +// "224.0.0.0/4", +// "240.0.0.0/4", +// "100.64.0.0/10", +// "198.18.0.0/15", +// "169.254.0.0/16", +// "192.88.99.0/24", +// "192.0.0.0/24", +// "192.0.2.0/24", +// "192.94.77.0/24", +// "192.94.78.0/24", +// "192.52.193.0/24", +// "192.12.109.0/24", +// "192.31.196.0/24", +// "192.0.0.0/29", +//} +// +//func init() { +// for _, cidr := range ReservedCIDRs { +// if _, ipnet, err := net.ParseCIDR(cidr); err == nil { +// reservedAddrRanges = append(reservedAddrRanges, ipnet) +// } +// } +//} +// +//// IsPrivateIP checks if the addr parameter is within one of the address ranges in the ReservedCIDRs slice. +//func IsPrivateIP(addr string) bool { +// ip := net.ParseIP(addr) +// if ip == nil { +// return false +// } +// +// var cidr string +// for _, block := range reservedAddrRanges { +// if block.Contains(ip) { +// cidr = block.String() +// break +// } +// } +// +// if cidr != "" { +// return true +// } +// return false +//} diff --git a/provider/provider_test.go b/provider/provider_test.go new file mode 100644 index 0000000..ceafda8 --- /dev/null +++ b/provider/provider_test.go @@ -0,0 +1,95 @@ +package provider + +import ( + "testing" + + "github.com/davecgh/go-spew/spew" +) + +var sampleToken = "d17ab13b0a3fccafefc932b9db95be45464339073100423336045977c0924491" + +func TestProviderList(t *testing.T) { + provider, err := InitProvider("do", sampleToken) + if err != nil { + t.Errorf("error") + } + provider.SSHPublicKey = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC0ZmMumN5GTKuPXVVqugz+6BZs6JyNaQGsPgsZk86uON//oXi5fkutsNu5IIPDZXph3P5NbUj2dPalNyzNY5jClPJeT0F/eowIhPeo5GfjmJlXR4TTgHSwPYrOQNop2w+xh2z5h4IXMEPVLpEsN67MuUjTKzRwirwPjigZS/gayhwfOfDsPaIwBhpoBGuR1x+Rzzxuiy7TToNoWhF6pT9qONoCtr0VrPMsmjVpEPKD/uTW/8KeFL0pb/9z18M4IlbtvkO0Y6RhrpFGNSmZTWc1eDsJpFJerrVd48rgx3aRHriijl4zX4GBhc0zjqJwv+nGTGFPJ9Tx/3kPMDUGna/f91VU7sL7YqeiSed8S0YcWfntYy64OknvMpN8VIoQ7WiJAkR3wPw+tL3ZduXXAiKHFTAiXev02mOvo2F2nQKdGS98lOH5m+zuUm8abYbyXYlGNEzz576ksb6nMWCSSXwhA5f4clPKaPmgBQFQMUtq6Wgb8Fjq2r1MpjIWwUvx84s= osmp-cloud" + provider.Prepare() + + provider.ListInstance() + spew.Dump(provider.Instances) +} + +func TestProviderCreate(t *testing.T) { + provider, err := InitProvider("do", sampleToken) + if err != nil { + t.Errorf("error ") + } + provider.SSHPublicKey = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC0ZmMumN5GTKuPXVVqugz+6BZs6JyNaQGsPgsZk86uON//oXi5fkutsNu5IIPDZXph3P5NbUj2dPalNyzNY5jClPJeT0F/eowIhPeo5GfjmJlXR4TTgHSwPYrOQNop2w+xh2z5h4IXMEPVLpEsN67MuUjTKzRwirwPjigZS/gayhwfOfDsPaIwBhpoBGuR1x+Rzzxuiy7TToNoWhF6pT9qONoCtr0VrPMsmjVpEPKD/uTW/8KeFL0pb/9z18M4IlbtvkO0Y6RhrpFGNSmZTWc1eDsJpFJerrVd48rgx3aRHriijl4zX4GBhc0zjqJwv+nGTGFPJ9Tx/3kPMDUGna/f91VU7sL7YqeiSed8S0YcWfntYy64OknvMpN8VIoQ7WiJAkR3wPw+tL3ZduXXAiKHFTAiXev02mOvo2F2nQKdGS98lOH5m+zuUm8abYbyXYlGNEzz576ksb6nMWCSSXwhA5f4clPKaPmgBQFQMUtq6Wgb8Fjq2r1MpjIWwUvx84s= osmp-cloud" + provider.Prepare() + + id, err := provider.CreateInstanceDO("example.com") + if err == nil { + provider.GetInstanceInfo(id) + } + spew.Dump(provider.Instances) +} + +func TestProviderDelete(t *testing.T) { + provider, err := InitProvider("do", sampleToken) + if err != nil { + t.Errorf("error ") + } + + provider.DeleteInstance("258443728") + spew.Dump(provider.Instances) +} + +func TestProviderAccount(t *testing.T) { + provider, err := InitProvider("do", sampleToken) + if err != nil { + t.Errorf("error ") + } + + provider.AccountDO() +} + +func TestProvider_DeleteSnapshot(t *testing.T) { + provider, err := InitProvider("do", sampleToken) + if err != nil { + t.Errorf("error ") + } + + provider.DeleteSnapShotDO("89053110") + provider.ListSnapshotDO() + spew.Dump(provider.OldSnapShotID) + spew.Dump(provider.SnapshotID) +} + +func TestProviderLN(t *testing.T) { + sampleToken = "6634197757df67b24753d0d241003ae09afd53bc7f9648c191e37acb61bfef37" + provider, err := InitProvider("linode", sampleToken) + if err != nil { + t.Errorf("error ") + } + provider.SSHPublicKey = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC0ZmMumN5GTKuPXVVqugz+6BZs6JyNaQGsPgsZk86uON//oXi5fkutsNu5IIPDZXph3P5NbUj2dPalNyzNY5jClPJeT0F/eowIhPeo5GfjmJlXR4TTgHSwPYrOQNop2w+xh2z5h4IXMEPVLpEsN67MuUjTKzRwirwPjigZS/gayhwfOfDsPaIwBhpoBGuR1x+Rzzxuiy7TToNoWhF6pT9qONoCtr0VrPMsmjVpEPKD/uTW/8KeFL0pb/9z18M4IlbtvkO0Y6RhrpFGNSmZTWc1eDsJpFJerrVd48rgx3aRHriijl4zX4GBhc0zjqJwv+nGTGFPJ9Tx/3kPMDUGna/f91VU7sL7YqeiSed8S0YcWfntYy64OknvMpN8VIoQ7WiJAkR3wPw+tL3ZduXXAiKHFTAiXev02mOvo2F2nQKdGS98lOH5m+zuUm8abYbyXYlGNEzz576ksb6nMWCSSXwhA5f4clPKaPmgBQFQMUtq6Wgb8Fjq2r1MpjIWwUvx84s= osmp-cloud" + //provider.GetSSHKey() + + provider.Prepare() + //id, _ := provider.CreateInstanceLN("new2") + //time.Sleep(90*time.Second) + //id := 29530061 + //provider.MountDiskLN(id) + provider.DeleteInstance("29530163") + //provider.AccountLN() + spew.Dump(provider.Instances) +} + +func TestProviderAccountAWS(t *testing.T) { + provider, err := InitProvider("aws", sampleToken) + if err != nil { + t.Errorf("error ") + } + + provider.AccountDO() +} diff --git a/server/auth.go b/server/auth.go new file mode 100644 index 0000000..3f7a11f --- /dev/null +++ b/server/auth.go @@ -0,0 +1,68 @@ +package server + +import ( + "time" + + // jwtware "github.com/gofiber/jwt/v2" + + "github.com/dgrijalva/jwt-go" + "github.com/gofiber/fiber/v2" +) + +// authen stuff + +func Login(c *fiber.Ctx) error { + type LoginInput struct { + Username string `json:"username"` + Password string `json:"password"` + } + var input LoginInput + if err := c.BodyParser(&input); err != nil { + return c.SendStatus(fiber.StatusUnauthorized) + } + user := input.Username + pass := input.Password + + dbUser := Opt.Client.Username + dbPass := Opt.Client.Password + + // auto pass if -A is specify + if Opt.Server.NoAuthen { + user = dbUser + pass = dbPass + } + + // Throws Unauthorized error + if dbUser != user || dbPass != pass { + return c.SendStatus(fiber.StatusUnauthorized) + } + + // Create the Claims + claims := jwt.MapClaims{ + "name": "Osmdeus Default", + "admin": true, + "exp": time.Now().Add(time.Hour * 24 * 30).Unix(), + } + + // Create token + token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) + + // Generate encoded token and send it as response. + t, err := token.SignedString([]byte(Opt.Server.JWTSecret)) + if err != nil { + return c.SendStatus(fiber.StatusInternalServerError) + } + return c.JSON(fiber.Map{"status": "success", "message": "Successfully login", "token": t}) + +} + +func jwtError(c *fiber.Ctx, err error) error { + if err.Error() == "Missing or malformed JWT" { + c.Status(fiber.StatusBadRequest) + return c.JSON(fiber.Map{"status": "error", "message": "Missing or malformed JWT", "data": nil}) + + } else { + c.Status(fiber.StatusUnauthorized) + return c.JSON(fiber.Map{"status": "error", "message": "Invalid or expired JWT", "data": nil}) + } +} diff --git a/server/builder.go b/server/builder.go new file mode 100644 index 0000000..eee0a13 --- /dev/null +++ b/server/builder.go @@ -0,0 +1,170 @@ +package server + +import ( + "fmt" + "github.com/gofiber/fiber/v2" + "github.com/j3ssie/osmedeus/libs" + "github.com/j3ssie/osmedeus/utils" + "os" + "path" + "strings" +) + +// UploadData data required in json form +type UploadData struct { + Data string `json:"data"` + Filename string `json:"filename"` +} + +// Upload testing authenticated connection +func Upload(c *fiber.Ctx) error { + var uploadData UploadData + + err := c.BodyParser(&uploadData) + if err != nil { + return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{ + "error": "Cannot parse JSON", + }) + } + + if uploadData.Filename == "" { + uploadData.Filename = utils.RandomString(6) + } + tmpFile := path.Base(utils.NormalizePath(uploadData.Filename)) + baseDir := fmt.Sprintf("/tmp/%v-input/", libs.BINARY) + if !utils.FolderExists(baseDir) { + os.MkdirAll(baseDir, 0755) + } + + filename := path.Join(baseDir, tmpFile) + data := uploadData.Data + + utils.WriteToFile(filename, data) + + return c.JSON(ResponseHTTP{ + Status: 200, + Data: fiber.Map{ + "filepath": filename, + }, + Type: "upload", + Message: "New Data Uploaded", + }) + +} + +// SaveTargets save upload data to /tmp/osm-input/data-osm-xxx.txt +func SaveTargets(targets []string) string { + tmpFile := fmt.Sprintf("data-%v-%v-%v.txt", utils.GetTS(), libs.BINARY, utils.RandomString(6)) + baseDir := fmt.Sprintf("/tmp/%v-input/", libs.BINARY) + if !utils.FolderExists(baseDir) { + os.MkdirAll(baseDir, 0755) + } + targetFile := path.Join(baseDir, tmpFile) + data := strings.Join(targets, "\n") + filename, err := utils.WriteToFile(targetFile, data) + if err != nil { + return "" + } + return filename +} + +// CommandBuilder build core command from API +func CommandBuilder(taskData *TaskData) string { + binary := fmt.Sprintf("%s scan", libs.BINARY) + if taskData.Distributed { + binary = fmt.Sprintf("%s cloud", libs.BINARY) + if taskData.Chunk { + binary = fmt.Sprintf("%s cloud --chunk", libs.BINARY) + } + } + + if taskData.ScanID != "" { + binary += fmt.Sprintf(" --sid %v ", taskData.ScanID) + } + + var command string + var workspace, concurrency, timeout, params, workflow, plugin, scanID string + + if len(taskData.TargetsList) > 0 { + taskData.TargetsFile = SaveTargets(taskData.TargetsList) + utils.DebugF("Save targets list to: %v", taskData.TargetsFile) + } + + // get workspace + if taskData.Workspace != "" { + taskData.Workspace = utils.CleanPath(taskData.Workspace) + workspace = fmt.Sprintf(" -w '%v'", taskData.Workspace) + } + + if taskData.Binary != "" { + binary = taskData.Binary + } + + //if taskData.RawName { + // binary = binary + " --rt " + //} + + if taskData.ViewOnly { + binary = binary + " --view-only " + } + + // default workflow is general + if taskData.WorkFlow != "" { + workflow = fmt.Sprintf(" -f '%v'", taskData.WorkFlow) + } + + // some mics options + if taskData.Timeout != "" { + timeout = fmt.Sprintf(" --timeout '%v'", taskData.Timeout) + } + if taskData.Concurrency > 0 { + concurrency = fmt.Sprintf(" -c %v", taskData.Concurrency) + } + if len(taskData.Params) > 0 { + for _, param := range taskData.Params { + // @NOTE replace ',' from request to ';;' first because corbra auto split ',' + if strings.Contains(param, ",") { + param = strings.Replace(param, ",", ";;", -1) + } + + if strings.HasPrefix(param, "'") && strings.HasSuffix(param, "'") { + params += fmt.Sprintf(" -p %s", param) + continue + } + params += fmt.Sprintf(" -p '%s'", param) + } + } + + if taskData.PluginName != "" { + plugin = fmt.Sprintf(" -m '%v'", taskData.PluginName) + } + + // override everything + if taskData.Command != "" { + return taskData.Command + } + + // mean general scan + if taskData.PluginName == "" { + command = fmt.Sprintf("%v %v -t %v %v%v%v%v", binary, workflow, taskData.Target, concurrency, timeout, workspace, params) + if taskData.TargetsFile != "" { + command = fmt.Sprintf("%v %v -T %v %v%v%v%v", binary, workflow, taskData.TargetsFile, concurrency, timeout, workspace, params) + } + command = strings.TrimSpace(command) + if taskData.Debug { + command = command + " --debug" + } + return command + } + + command = fmt.Sprintf("%v %v -t %v %v %v%v%v%v", binary, plugin, taskData.Target, scanID, concurrency, timeout, workspace, params) + if taskData.TargetsFile != "" { + command = fmt.Sprintf("%v %v -t %v %v %v%v%v%v", binary, plugin, taskData.TargetsFile, scanID, concurrency, timeout, workspace, params) + } + + command = strings.TrimSpace(command) + if taskData.Debug { + command = command + " --debug" + } + return command +} diff --git a/server/docs/docs.go b/server/docs/docs.go new file mode 100644 index 0000000..43b1bb3 --- /dev/null +++ b/server/docs/docs.go @@ -0,0 +1,120 @@ +// GENERATED BY THE COMMAND ABOVE; DO NOT EDIT +// This file was generated by swaggo/swag + +package docs + +import ( + "bytes" + "encoding/json" + "strings" + + "github.com/alecthomas/template" + "github.com/swaggo/swag" +) + +var doc = `{ + "schemes": {{ marshal .Schemes }}, + "swagger": "2.0", + "info": { + "description": "{{.Description}}", + "title": "{{.Title}}", + "contact": {}, + "version": "{{.Version}}" + }, + "host": "{{.Host}}", + "basePath": "{{.BasePath}}", + "paths": { + "/v1/books": { + "get": { + "description": "Get all books", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "books" + ], + "summary": "Get all books", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/handler.ResponseHTTP" + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/handler.ResponseHTTP" + } + } + } + } + } + }, + "definitions": { + "handler.ResponseHTTP": { + "type": "object", + "properties": { + "data": { + "type": "object" + }, + "message": { + "type": "string" + }, + "success": { + "type": "boolean" + } + } + } + } +}` + +type swaggerInfo struct { + Version string + Host string + BasePath string + Schemes []string + Title string + Description string +} + +// SwaggerInfo holds exported Swagger Info so clients can modify it +var SwaggerInfo = swaggerInfo{ + Version: "", + Host: "", + BasePath: "", + Schemes: []string{}, + Title: "", + Description: "", +} + +type s struct{} + +func (s *s) ReadDoc() string { + sInfo := SwaggerInfo + sInfo.Description = strings.Replace(sInfo.Description, "\n", "\\n", -1) + + t, err := template.New("swagger_info").Funcs(template.FuncMap{ + "marshal": func(v interface{}) string { + a, _ := json.Marshal(v) + return string(a) + }, + }).Parse(doc) + if err != nil { + return doc + } + + var tpl bytes.Buffer + if err := t.Execute(&tpl, sInfo); err != nil { + return doc + } + + return tpl.String() +} + +func init() { + swag.Register(swag.Name, &s{}) +} diff --git a/server/docs/swagger.json b/server/docs/swagger.json new file mode 100644 index 0000000..041a76b --- /dev/null +++ b/server/docs/swagger.json @@ -0,0 +1,53 @@ +{ + "swagger": "2.0", + "info": { + "contact": {} + }, + "paths": { + "/v1/books": { + "get": { + "description": "Get all books", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "books" + ], + "summary": "Get all books", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/handler.ResponseHTTP" + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/handler.ResponseHTTP" + } + } + } + } + } + }, + "definitions": { + "handler.ResponseHTTP": { + "type": "object", + "properties": { + "data": { + "type": "object" + }, + "message": { + "type": "string" + }, + "success": { + "type": "boolean" + } + } + } + } +} \ No newline at end of file diff --git a/server/docs/swagger.yaml b/server/docs/swagger.yaml new file mode 100644 index 0000000..9b3a48a --- /dev/null +++ b/server/docs/swagger.yaml @@ -0,0 +1,33 @@ +definitions: + handler.ResponseHTTP: + properties: + data: + type: object + message: + type: string + success: + type: boolean + type: object +info: + contact: { } +paths: + /v1/books: + get: + consumes: + - application/json + description: Get all books + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/handler.ResponseHTTP' + "503": + description: Service Unavailable + schema: + $ref: '#/definitions/handler.ResponseHTTP' + summary: Get all books + tags: + - books +swagger: "2.0" diff --git a/server/mics.go b/server/mics.go new file mode 100644 index 0000000..ba035e4 --- /dev/null +++ b/server/mics.go @@ -0,0 +1,100 @@ +package server + +import ( + "fmt" + "path" + "path/filepath" + "strings" + + "github.com/gofiber/fiber/v2" + "github.com/j3ssie/osmedeus/core" + "github.com/j3ssie/osmedeus/execution" + "github.com/j3ssie/osmedeus/libs" +) + +func Process(c *fiber.Ctx) error { + processes := execution.GetOsmProcess("") + return c.JSON(ResponseHTTP{ + Status: 200, + Data: processes, + Type: "processes", + Total: len(processes), + Message: "List all osm process", + }) +} + +func RawWorkspace(c *fiber.Ctx) error { + return c.JSON(ResponseHTTP{ + Status: 200, + Data: fiber.Map{ + "storages": fmt.Sprintf("/%s/storages/", Opt.Server.StaticPrefix), + "workspaces": fmt.Sprintf("/%s/workspaces/", Opt.Server.StaticPrefix), + "logs": fmt.Sprintf("/%s/logs/", Opt.Server.StaticPrefix), + }, + Type: "raw", + Message: "Raw directory", + }) +} + +func ListFlows(c *fiber.Ctx) error { + flows := core.ListFlow(Opt) + if len(flows) == 0 { + return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{ + "error": "Can't list workflow", + }) + } + + var result []map[string]string + + for _, flow := range flows { + if flow != "" { + item := make(map[string]string) + item["name"] = strings.TrimSuffix(filepath.Base(flow), ".yaml") + + // get modules + Opt.Flow.Type = strings.TrimSuffix(item["name"], path.Ext(item["name"])) + rawModules := core.ListModules(Opt) + var modules []string + for _, module := range rawModules { + if module != "" { + modules = append(modules, strings.TrimSuffix(filepath.Base(module), ".yaml")) + } + } + + item["desc"] = "" + parsedFlow, err := core.ParseFlow(flow) + if err == nil { + item["desc"] = parsedFlow.Desc + } + + item["modules"] = strings.Join(modules, ",") + result = append(result, item) + + } + } + + return c.JSON(ResponseHTTP{ + Status: 200, + Data: result, + Total: len(flows), + Type: "flows", + Message: "Workflows Listing", + }) +} + +func HelperMessage(c *fiber.Ctx) error { + message := fmt.Sprintf(` +[*] Visit this page for complete Usage: %s +`, libs.DOCS) + + return c.JSON(ResponseHTTP{ + Status: 200, + Data: fiber.Map{ + "version": libs.VERSION, + "doc": libs.DOCS, + "message": message, + }, + Type: "helper", + Message: "Helper message", + }) +} diff --git a/server/ping.go b/server/ping.go new file mode 100644 index 0000000..9852e82 --- /dev/null +++ b/server/ping.go @@ -0,0 +1,41 @@ +package server + +import ( + "github.com/gofiber/fiber/v2" + "github.com/j3ssie/osmedeus/libs" +) + +// Ping is a function to get all books data from database +// @Summary Get all books +// @Description Get all books +// @Tags books +// @Accept json +// @Produce json +// @Success 200 {object} ResponseHTTP{} +// @Failure 503 {object} ResponseHTTP{} +// @Router /v1/books [get] +func Ping(c *fiber.Ctx) error { + return c.JSON(ResponseHTTP{ + Status: 200, + Message: "pong", + }) +} + +// Health is a function to get all books data from database +// @Summary Get all books +// @Description Get all books +// @Tags books +// @Accept json +// @Produce json +// @Success 200 {object} ResponseHTTP{} +// @Failure 503 {object} ResponseHTTP{} +// @Router /v1/books [get] +func Health(c *fiber.Ctx) error { + return c.JSON(ResponseHTTP{ + Status: 200, + Data: fiber.Map{ + "version": libs.VERSION, + }, + Message: "server is up", + }) +} diff --git a/server/router.go b/server/router.go new file mode 100644 index 0000000..033c5af --- /dev/null +++ b/server/router.go @@ -0,0 +1,152 @@ +package server + +import ( + "crypto/tls" + "fmt" + "log" + "net" + "net/http" + "os" + "path" + + "github.com/fatih/color" + "github.com/gofiber/fiber/v2" + "github.com/gofiber/fiber/v2/middleware/cors" + jwtware "github.com/gofiber/jwt/v2" + "github.com/j3ssie/osmedeus/libs" + "github.com/j3ssie/osmedeus/utils" + + "github.com/gofiber/fiber/v2/middleware/filesystem" + "github.com/gofiber/fiber/v2/middleware/logger" +) + +// var DB *gorm.DB +var Opt libs.Options + +func StartServer(options libs.Options) { + Opt = options + var err error + + if options.Server.NoAuthen { + fmt.Fprintf(os.Stderr, color.RedString("[Critical] The server is currently being executed %v mechanism enabled.\n", color.HiYellowString("WITHOUT ANY AUTHENTICATION"))) + } + + app := fiber.New(fiber.Config{ + Prefork: options.Server.PreFork, + }) + app.Use(cors.New()) + SetupRoutes(app) + + // mean enable SSL + var enableSSL bool + var ln net.Listener + if !options.Server.DisableSSL { + err = EnableSSL(options) + if err == nil { + enableSSL = true + } + cer, err := tls.LoadX509KeyPair(options.Server.CertFile, options.Server.KeyFile) + if err != nil { + enableSSL = false + utils.ErrorF("error create ssl listener: %v", err) + } + config := &tls.Config{Certificates: []tls.Certificate{cer}} + + // Create custom listener + ln, err = tls.Listen("tcp", options.Server.Bind, config) + if err != nil { + utils.ErrorF("error create ssl listener: %v", err) + enableSSL = false + } + } + + if enableSSL { + utils.GoodF("Web UI available at: %v ", color.HiMagentaString("https://%v/ui/", options.Server.Bind)) + utils.GoodF("Static Content available at: %v", color.HiMagentaString("https://%v/%s/workspaces/", options.Server.Bind, Opt.Server.StaticPrefix)) + log.Fatal(app.Listener(ln)) + } else { + utils.GoodF("Web UI available at: http://%v/ui/", options.Server.Bind) + log.Fatal(app.Listen(options.Server.Bind)) + } +} + +// SetupRoutes setup router api +func SetupRoutes(app *fiber.App) { + // for UI + app.Static("/ui", Opt.Server.UIPath) + app.Get("/ui/*", func(ctx *fiber.Ctx) error { + return ctx.SendFile(path.Join(Opt.Server.UIPath, "index.html")) + }) + + // for static report file + app.Use(fmt.Sprintf("%s/workspaces/", Opt.Server.StaticPrefix), filesystem.New(filesystem.Config{ + Root: http.Dir(Opt.Env.WorkspacesFolder), + Browse: !Opt.Server.DisableWorkspaceListing, + MaxAge: 3600, + NotFoundFile: "", + })) + app.Use(fmt.Sprintf("%s/storages/", Opt.Server.StaticPrefix), filesystem.New(filesystem.Config{ + Root: http.Dir(Opt.Env.StoragesFolder), + Browse: !Opt.Server.DisableWorkspaceListing, + MaxAge: 3600, + NotFoundFile: "", + })) + + // for swagger document + //app.Get("/docs/*", swagger.Handler) // default + //app.Get("/docs/*", swagger.New(swagger.Config{ // custom + // URL: "https://osmp.io/doc.json", + // DeepLinking: false, + //})) + + app.Get("/ping", Ping) + api := app.Group("/api", logger.New()) + api.Post("/login", Login) + + // disable JWT Middleware when -A is set + if !Opt.Server.NoAuthen { + app.Use(jwtware.New(jwtware.Config{ + SigningKey: []byte(Opt.Server.JWTSecret), + Filter: nil, + SuccessHandler: nil, + ErrorHandler: jwtError, + SigningKeys: nil, + SigningMethod: "", + ContextKey: "", + Claims: nil, + TokenLookup: "", + AuthScheme: "Osmedeus", + })) + } + + // Middleware + osmp := api.Group("/osmp") + // /api/osmp/health + osmp.Get("/health", Health) + + // core API e.g: /api/osmp/workspaces + osmp.Get("/workspaces", ListWorkspaces) + osmp.Get("/workspace/:wsname/", WorkspaceDetail) + osmp.Get("/scans", ListAllScan) + osmp.Delete("/delete/:wsname/", DeleteWorkspace) + + osmp.Get("/ps", Process) + osmp.Get("/raw", RawWorkspace) + osmp.Get("/flows", ListFlows) + osmp.Get("/help", HelperMessage) + + //api.Use(basicauth.New(basicauth.Config{ + // Users: map[string]string{ + // Opt.Client.Username: Opt.Client.Password, + // }, + // Realm: "Forbidden", + // Unauthorized: func(c *fiber.Ctx) error { + // return c.SendString("404 not found") + // }, + //})) + // + + // execute endpoints + osmp.Post("/execute", NewScan) + osmp.Post("/upload", Upload) +} diff --git a/server/scan.go b/server/scan.go new file mode 100644 index 0000000..42ec9d6 --- /dev/null +++ b/server/scan.go @@ -0,0 +1,130 @@ +package server + +import ( + "github.com/gofiber/fiber/v2" + "github.com/j3ssie/osmedeus/utils" +) + +// TaskData data required in json form +type TaskData struct { + MasterPassword string `json:"password"` + Binary string `json:"binary"` + // override everything below + Command string `json:"command"` + + // these two not be blank when run with plugins + WorkFlow string `json:"workflow"` + PluginName string `json:"plugin"` + ScanID string `json:"scan_id"` + + // for select scan + task + Workspace string `json:"workspace"` + Target string `json:"target"` + TargetsList []string `json:"targets"` + TargetsFile string `json:"targets_file"` + + AliveAssets bool `json:"alive_assets"` // skip targets part and select the assets from DB + AllAssets bool `json:"all_assets"` // skip targets part and select the assets from DB + + // just more mics info for custom command later + Params []string `json:"params"` + Timeout string `json:"timeout"` + Concurrency int `json:"concurrency"` + + // enable distributed scan + Distributed bool `json:"distributed"` + + // for chunk mode only + Threads int `json:"threads"` + Chunk bool `json:"chunk"` + TargetAsFile bool `json:"as_file"` + + // only select record not run the command + RawName bool `json:"RawName"` + WildCard bool `json:"wildcard"` + ViewOnly bool `json:"view_only"` + Debug bool `json:"debug"` + Test bool `json:"test"` +} + +// NewScan new scan +func NewScan(c *fiber.Ctx) error { + var taskData TaskData + var invalid bool + + err := c.BodyParser(&taskData) + if err != nil { + invalid = true + } + + if invalid { + return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{ + "error": "Cannot parse JSON", + }) + } + + taskData.Command = CommandBuilder(&taskData) + // get workspace if we didn't have one + if taskData.Workspace == "" { + taskData.Workspace = utils.CleanPath(taskData.Target) + } + utils.InforF("Running command: %v", taskData.Command) + + if !taskData.Test { + go func() { + utils.RunOSCommand(taskData.Command) + return + }() + } + + return c.JSON(ResponseHTTP{ + Status: 200, + Data: fiber.Map{ + "command": taskData.Command, + }, + Type: "new-scan", + Message: "New Scan Imported", + }) +} + +// NewScanCloud new scan +func NewScanCloud(c *fiber.Ctx) error { + var taskData TaskData + var invalid bool + + err := c.BodyParser(&taskData) + if err != nil { + invalid = true + } + + if invalid { + return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{ + "error": "Cannot parse JSON", + }) + } + + taskData.Command = CommandBuilder(&taskData) + // get workspace if we didn't have one + if taskData.Workspace == "" { + taskData.Workspace = utils.CleanPath(taskData.Target) + } + utils.InforF("Running command: %v", taskData.Command) + + if !taskData.Test { + go func() { + utils.RunOSCommand(taskData.Command) + return + }() + } + + return c.JSON(ResponseHTTP{ + Status: 200, + Data: fiber.Map{ + "input": taskData.Target, + "scan_id": taskData.ScanID, + "workflow": taskData.WorkFlow, + }, + Type: "new-scan", + Message: "New Scan Imported", + }) +} diff --git a/server/ssl.go b/server/ssl.go new file mode 100644 index 0000000..53da63a --- /dev/null +++ b/server/ssl.go @@ -0,0 +1,191 @@ +package server + +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Generate a self-signed X.509 certificate for a TLS server. Outputs to +// 'cert.pem' and 'key.pem' and will overwrite existing files. + +import ( + "crypto/ecdsa" + "crypto/ed25519" + "crypto/elliptic" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "github.com/j3ssie/osmedeus/libs" + "github.com/j3ssie/osmedeus/utils" + "log" + "math/big" + "net" + "os" + "strings" + "time" +) + +func publicKey(priv interface{}) interface{} { + switch k := priv.(type) { + case *rsa.PrivateKey: + return &k.PublicKey + case *ecdsa.PrivateKey: + return &k.PublicKey + case ed25519.PrivateKey: + return k.Public().(ed25519.PublicKey) + default: + return nil + } +} + +// EnableSSL do some check when ssl enabled +func EnableSSL(options libs.Options) error { + if options.Server.DisableSSL { + return nil + } + + if utils.FileExists(options.Server.CertFile) && utils.FileExists(options.Server.KeyFile) { + return nil + } + + ok := GenerateSSL(options.Server.CertFile, options.Server.KeyFile) + if ok { + return nil + } + utils.ErrorF("error create ssl key at: %v", options.Server.CertFile) + return fmt.Errorf("error create SSL Key") +} + +// GenerateSSL generate SSL key +func GenerateSSL(certFile string, keyFile string) bool { + host := "localhost" // "Comma-separated hostnames and IPs to generate a certificate for + validFrom := "" // "Creation date formatted as Jan 1 15:04:05 2011 + validFor := 365 * 24 * time.Hour // "Duration that certificate is valid for + isCA := false // "whether this cert should be its own Certificate Authority + rsaBits := 4096 // "Size of RSA key to generate. Ignored if --ecdsa-curve is set + ecdsaCurve := "P256" // "ECDSA curve to use to generate a key. Valid values are P224, P256 (recommended), P384, P521 + ed25519Key := false // "Generate an Ed25519 key" + + var priv interface{} + var err error + switch ecdsaCurve { + case "": + if ed25519Key { + _, priv, err = ed25519.GenerateKey(rand.Reader) + } else { + priv, err = rsa.GenerateKey(rand.Reader, rsaBits) + } + case "P224": + priv, err = ecdsa.GenerateKey(elliptic.P224(), rand.Reader) + case "P256": + priv, err = ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + case "P384": + priv, err = ecdsa.GenerateKey(elliptic.P384(), rand.Reader) + case "P521": + priv, err = ecdsa.GenerateKey(elliptic.P521(), rand.Reader) + default: + log.Fatalf("Unrecognized elliptic curve: %q", ecdsaCurve) + } + if err != nil { + log.Fatalf("Failed to generate private key: %v", err) + } + + // ECDSA, ED25519 and RSA subject keys should have the DigitalSignature + // KeyUsage bits set in the x509.Certificate template + keyUsage := x509.KeyUsageDigitalSignature + // Only RSA subject keys should have the KeyEncipherment KeyUsage bits set. In + // the context of TLS this KeyUsage is particular to RSA key exchange and + // authentication. + if _, isRSA := priv.(*rsa.PrivateKey); isRSA { + keyUsage |= x509.KeyUsageKeyEncipherment + } + + var notBefore time.Time + if len(validFrom) == 0 { + notBefore = time.Now() + } else { + notBefore, err = time.Parse("Jan 2 15:04:05 2006", validFrom) + if err != nil { + log.Fatalf("Failed to parse creation date: %v", err) + } + } + + notAfter := notBefore.Add(validFor) + + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) + serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) + if err != nil { + log.Fatalf("Failed to generate serial number: %v", err) + } + + template := x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{ + Organization: []string{"Not Localhost"}, + }, + NotBefore: notBefore, + NotAfter: notAfter, + + KeyUsage: keyUsage, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + } + + hosts := strings.Split(host, ",") + for _, h := range hosts { + if ip := net.ParseIP(h); ip != nil { + template.IPAddresses = append(template.IPAddresses, ip) + } else { + template.DNSNames = append(template.DNSNames, h) + } + } + + if isCA { + template.IsCA = true + template.KeyUsage |= x509.KeyUsageCertSign + } + + derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, publicKey(priv), priv) + if err != nil { + utils.ErrorF("Failed to create certificate: %v", err) + return false + } + + certOut, err := os.Create(certFile) + if err != nil { + utils.ErrorF("Failed to open %s for writing: %v", certFile, err) + return false + } + if err := pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil { + utils.ErrorF("Failed to write data to cert.pem: %v", err) + return false + } + if err := certOut.Close(); err != nil { + utils.ErrorF("Error closing cert.pem: %v", err) + return false + } + //log.Print("wrote cert.pem\n") + + keyOut, err := os.OpenFile(keyFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) + if err != nil { + utils.ErrorF("Failed to open %sfor writing: %v", keyFile, err) + return false + } + privBytes, err := x509.MarshalPKCS8PrivateKey(priv) + if err != nil { + utils.ErrorF("Unable to marshal private key: %v", err) + return false + } + if err := pem.Encode(keyOut, &pem.Block{Type: "PRIVATE KEY", Bytes: privBytes}); err != nil { + utils.ErrorF("Failed to write data to key.pem: %v", err) + return false + } + if err := keyOut.Close(); err != nil { + utils.ErrorF("Error closing key.pem: %v", err) + return false + } + //log.Print("wrote key.pem\n") + return true +} diff --git a/server/workspace.go b/server/workspace.go new file mode 100644 index 0000000..3a1746e --- /dev/null +++ b/server/workspace.go @@ -0,0 +1,118 @@ +package server + +import ( + "fmt" + "os" + "path" + "strings" + + "github.com/gofiber/fiber/v2" + "github.com/j3ssie/osmedeus/database" + "github.com/j3ssie/osmedeus/utils" + "github.com/thoas/go-funk" +) + +// ResponseHTTP represents response body of this API +type ResponseHTTP struct { + Status int `json:"status"` + Data interface{} `json:"data"` + Type string `json:"type,omitempty"` + Total int `json:"total,omitempty"` + Message string `json:"message"` +} + +// Workspace is a function to get all books data from database +// @Summary Get all books +// @Description Get all books +// @Tags books +// @Accept json +// @Produce json +// @Success 200 {object} ResponseHTTP{} +// @Router /v1/workspaces [get] +func ListWorkspaces(c *fiber.Ctx) error { + workspaces := database.GetAllScan(Opt) + return c.JSON(ResponseHTTP{ + Status: 200, + Data: workspaces, + Type: "workspaces", + Total: len(workspaces), + Message: "List all of Workspaces", + }) +} + +func WorkspaceDetail(c *fiber.Ctx) error { + wsname := c.Params("wsname") + workspace := database.GetSingleScan(wsname, Opt) + + // make a reports map + rawReports := workspace.Target.Reports + reports := make(map[string][]string) + + for _, report := range rawReports { + // replace the home folder first + if strings.Contains(report.ReportPath, workspace.Target.Workspace) { + // /root/workspaces-osmedeus/ + homeFolder := strings.Split(report.ReportPath, workspace.Target.Workspace)[0] + report.ReportPath = strings.ReplaceAll(report.ReportPath, homeFolder, Opt.Env.WorkspacesFolder+"/") + } + + if utils.FileLength(report.ReportPath) == 0 { + if utils.FolderExists(report.ReportPath) && utils.FolderLength(report.ReportPath) == 0 { + continue + } else { + continue + } + } + + if strings.HasPrefix(report.ReportPath, Opt.Env.WorkspacesFolder) { + report.ReportPath = strings.ReplaceAll(report.ReportPath, Opt.Env.WorkspacesFolder, fmt.Sprintf("/%v/workspaces", Opt.Server.StaticPrefix)) + } else if strings.HasPrefix(report.ReportPath, Opt.Env.StoragesFolder) { + report.ReportPath = strings.ReplaceAll(report.ReportPath, Opt.Env.StoragesFolder, fmt.Sprintf("/%v/storages", Opt.Server.StaticPrefix)) + } + + if !funk.Contains(reports[report.Module], report.ReportPath) { + reports[report.Module] = append(reports[report.Module], report.ReportPath) + } + } + + return c.JSON(ResponseHTTP{ + Status: 200, + Data: fiber.Map{ + "workspace": workspace, + "reports": reports, + }, + Type: "workspace", + Message: "Workspace Detail ", + }) +} + +func ListAllScan(c *fiber.Ctx) error { + scan := database.GetScanProgress(Opt) + return c.JSON(ResponseHTTP{ + Status: 200, + Data: scan, + Type: "scans", + Total: len(scan), + Message: "List all the scan process", + }) +} + +func DeleteWorkspace(c *fiber.Ctx) error { + wsname := c.Params("wsname") + wsDir := path.Join(Opt.Env.WorkspacesFolder, utils.NormalizePath(wsname)) + if !utils.FolderExists(wsDir) { + wsDir = path.Join(Opt.Env.WorkspacesFolder, utils.StripPath(wsname)) + if !utils.FolderExists(wsDir) { + return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{ + "error": "workspace didn't exist", + }) + } + } + + os.RemoveAll(wsDir) + return c.JSON(ResponseHTTP{ + Status: 200, + Type: "delete", + Message: "Workspace Deleted", + }) +} diff --git a/test-workflows/flow-with-params.yaml b/test-workflows/flow-with-params.yaml new file mode 100644 index 0000000..0a7fb01 --- /dev/null +++ b/test-workflows/flow-with-params.yaml @@ -0,0 +1,11 @@ +name: flow-with-params +desc: run normal routine +type: sample + +params: + - firstTimeout: '5s' + +routines: + - modules: + - timeout-module + diff --git a/test-workflows/general.yaml b/test-workflows/general.yaml new file mode 100644 index 0000000..9510506 --- /dev/null +++ b/test-workflows/general.yaml @@ -0,0 +1,31 @@ +name: general +desc: run normal routine +type: general +validator: domain + +routines: + - modules: + - subdomain + - modules: + - probing + - modules: + - fingerprint + - modules: + - portscan + - modules: + - ipspace + - modules: + - archive + - modules: + - spider + - modules: + - dirbscan + - modules: + - vulnscan + - modules: + - cloudbrute + - modules: + - credintel + - modules: + - summary + diff --git a/test-workflows/general/archive.yaml b/test-workflows/general/archive.yaml new file mode 100644 index 0000000..bba8b29 --- /dev/null +++ b/test-workflows/general/archive.yaml @@ -0,0 +1,10 @@ +name: archive +desc: archive + +params: + - testData: "~/go/src/github.com/j3ssie/osmedeus/test-data" + +steps: + # get data from cdn + - scripts: + - ImportArchive("{{testData}}/{{Workspace}}/archive/archive.txt") \ No newline at end of file diff --git a/test-workflows/general/cloudbrute.yaml b/test-workflows/general/cloudbrute.yaml new file mode 100644 index 0000000..dc5337d --- /dev/null +++ b/test-workflows/general/cloudbrute.yaml @@ -0,0 +1,10 @@ +name: cloudbrute +desc: cloudbrute + +params: + - testData: "~/go/src/github.com/j3ssie/osmedeus/test-data" + +steps: + # get data from cdn + - scripts: + - ImportCloudBrute("{{testData}}/{{Workspace}}/clouds-data.txt") diff --git a/test-workflows/general/credintel.yaml b/test-workflows/general/credintel.yaml new file mode 100644 index 0000000..6fe2ba4 --- /dev/null +++ b/test-workflows/general/credintel.yaml @@ -0,0 +1,10 @@ +name: credintel +desc: credintel + +params: + - testData: "~/go/src/github.com/j3ssie/osmedeus/test-data" + +steps: + # get data from cdn + - scripts: + - ImportCred("{{testData}}/{{Workspace}}/new-intel-cred.txt") diff --git a/test-workflows/general/dirbscan.yaml b/test-workflows/general/dirbscan.yaml new file mode 100644 index 0000000..7ad2bb2 --- /dev/null +++ b/test-workflows/general/dirbscan.yaml @@ -0,0 +1,10 @@ +name: dirbscan +desc: dirbscan + +params: + - testData: "~/go/src/github.com/j3ssie/osmedeus/test-data" + +steps: + # get data from cdn + - scripts: + - ImportDirectoryJson("{{testData}}/{{Workspace}}/directory/directory-json.txt") diff --git a/test-workflows/general/fingerprint.yaml b/test-workflows/general/fingerprint.yaml new file mode 100644 index 0000000..84fc4c2 --- /dev/null +++ b/test-workflows/general/fingerprint.yaml @@ -0,0 +1,13 @@ +name: fingerprint +desc: Scanning for subdomain + +params: + - testData: "~/go/src/github.com/j3ssie/osmedeus/test-data" + +steps: + # get data from cdn + - scripts: + - ImportTech("{{testData}}/{{Workspace}}/fingerprint/{{Workspace}}-technologies.txt") + - ImportHTTPJson("{{testData}}/{{Workspace}}/fingerprint/{{Workspace}}-technologies.txt") + - ImportScreenShotJson("{{testData}}/{{Workspace}}/screenshot/goverview/screenshot-summary.txt") + - ImportHTTPJson("{{testData}}/{{Workspace}}/screenshot/goverview/content-summary.txt") \ No newline at end of file diff --git a/test-workflows/general/ipspace.yaml b/test-workflows/general/ipspace.yaml new file mode 100644 index 0000000..ef22a7d --- /dev/null +++ b/test-workflows/general/ipspace.yaml @@ -0,0 +1,10 @@ +name: ipspace +desc: ipspace + +params: + - testData: "~/go/src/github.com/j3ssie/osmedeus/test-data" + +steps: + # get data from cdn + - scripts: + - ImportIPRange("{{testData}}/{{Workspace}}/ipspace/ipspace-summary.txt") \ No newline at end of file diff --git a/test-workflows/general/portscan.yaml b/test-workflows/general/portscan.yaml new file mode 100644 index 0000000..3f9a8dc --- /dev/null +++ b/test-workflows/general/portscan.yaml @@ -0,0 +1,10 @@ +name: portscan +desc: Scanning for subdomain + +params: + - testData: "~/go/src/github.com/j3ssie/osmedeus/test-data" + +steps: + # get data from cdn + - scripts: + - ImportPortJson("{{testData}}/{{Workspace}}/portscan/portscan-json.txt") diff --git a/test-workflows/general/probing.yaml b/test-workflows/general/probing.yaml new file mode 100644 index 0000000..898d5d7 --- /dev/null +++ b/test-workflows/general/probing.yaml @@ -0,0 +1,10 @@ +name: probing +desc: Scanning for subdomain + +params: + - testData: "~/go/src/github.com/j3ssie/osmedeus/test-data" + +steps: + # get data from cdn + - scripts: + - ImportDns("{{testData}}/{{Workspace}}/probing/dns-{{Workspace}}.txt") diff --git a/test-workflows/general/spider.yaml b/test-workflows/general/spider.yaml new file mode 100644 index 0000000..ecebbd4 --- /dev/null +++ b/test-workflows/general/spider.yaml @@ -0,0 +1,10 @@ +name: spider +desc: spider + +params: + - testData: "~/go/src/github.com/j3ssie/osmedeus/test-data" + +steps: + # get data from cdn + - scripts: + - ImportLinks("{{testData}}/{{Workspace}}/linkfinding/links-json.txt") diff --git a/test-workflows/general/subdomain.yaml b/test-workflows/general/subdomain.yaml new file mode 100644 index 0000000..7c939f8 --- /dev/null +++ b/test-workflows/general/subdomain.yaml @@ -0,0 +1,10 @@ +name: subdomain +desc: Scanning for subdomain + +params: + - testData: "~/go/src/github.com/j3ssie/osmedeus/test-data" + +steps: + # get data from cdn + - scripts: + - ImportSubdomain("{{testData}}/{{Workspace}}/subdomain/final-{{Workspace}}.txt") diff --git a/test-workflows/general/summary.yaml b/test-workflows/general/summary.yaml new file mode 100644 index 0000000..345cb77 --- /dev/null +++ b/test-workflows/general/summary.yaml @@ -0,0 +1,10 @@ +name: summary +desc: summary + +params: + - testData: "~/go/src/github.com/j3ssie/osmedeus/test-data" + +steps: + # get data from cdn + - scripts: + - SummaryTarget() \ No newline at end of file diff --git a/test-workflows/general/vulnscan.yaml b/test-workflows/general/vulnscan.yaml new file mode 100644 index 0000000..0057805 --- /dev/null +++ b/test-workflows/general/vulnscan.yaml @@ -0,0 +1,11 @@ +name: vulnscan +desc: vulnscan + +params: + - testData: "~/go/src/github.com/j3ssie/osmedeus/test-data" + +steps: + # get data from cdn + - scripts: + - ImportJaelesVulnJson("{{testData}}/{{Workspace}}/vuln/active/jaeles-summary.txt") + - ImportNucleiVulnJson("{{testData}}/{{Workspace}}/vuln/nuclei-sto.txt") diff --git a/test-workflows/parallel.yaml b/test-workflows/parallel.yaml new file mode 100644 index 0000000..ff386d2 --- /dev/null +++ b/test-workflows/parallel.yaml @@ -0,0 +1,14 @@ +name: flow-with-params +desc: run normal routine +type: sample + +params: + - firstTimeout: '5s' + +routines: + - modules: + - parallel + - parallel2 + - modules: + - timeout-module + diff --git a/test-workflows/pre-run-cloud.yaml b/test-workflows/pre-run-cloud.yaml new file mode 100644 index 0000000..8ac58d6 --- /dev/null +++ b/test-workflows/pre-run-cloud.yaml @@ -0,0 +1,17 @@ +name: test +desc: testing workflow +type: test +validator: domain + +params: + - repo: "test-module" + +local_pre_run: + - 'RRSync("root@{{RemoteIP}}", "/tmp/sam/", "/tmp/sam/")' + - ExecCmd('echo --> {{repo}}') + +routines: + - modules: + - sub-test + - modules: + - prob-test diff --git a/test-workflows/sample/parallel.yaml b/test-workflows/sample/parallel.yaml new file mode 100644 index 0000000..f3862ca --- /dev/null +++ b/test-workflows/sample/parallel.yaml @@ -0,0 +1,20 @@ +name: parallel1 +desc: Run dirbscan scan on list of HTTP file + +params: + - limit: '5000' + +steps: + - label: 'Start step 111' + commands: + - "echo '---> {{Target}} '" + - "seq {{limit}} > /tmp/source.txt" + + - source: '/tmp/source.txt' + threads: '200' + commands: + - "echo '---> [[.line]]'" + # - "sleep 1 && echo '---> done [[.line]]'" + + - commands: + - "echo 'done parallel 111111'" \ No newline at end of file diff --git a/test-workflows/sample/parallel2.yaml b/test-workflows/sample/parallel2.yaml new file mode 100644 index 0000000..093ac83 --- /dev/null +++ b/test-workflows/sample/parallel2.yaml @@ -0,0 +1,12 @@ +name: parallel2 +desc: Run dirbscan scan on list of HTTP file + +steps: + - label: 'Start step 222' + commands: + - "sleep 5" + - "echo '==>>>> {{Target}} '" + + - commands: + - "echo 'done parallel 2222'" + \ No newline at end of file diff --git a/test-workflows/sample/timeout-module.yaml b/test-workflows/sample/timeout-module.yaml new file mode 100644 index 0000000..99df50c --- /dev/null +++ b/test-workflows/sample/timeout-module.yaml @@ -0,0 +1,6 @@ +name: timeout-module +desc: + +steps: + - commands: + - "echo '==> Done here <==" \ No newline at end of file diff --git a/test-workflows/serial.yaml b/test-workflows/serial.yaml new file mode 100644 index 0000000..e439d22 --- /dev/null +++ b/test-workflows/serial.yaml @@ -0,0 +1,12 @@ +name: flow-with-params +desc: run normal routine +type: sample + +params: + - firstTimeout: '5s' + +routines: + - modules: + - parallel + + diff --git a/test-workflows/test-module/loop-step.yaml b/test-workflows/test-module/loop-step.yaml new file mode 100644 index 0000000..0c7e421 --- /dev/null +++ b/test-workflows/test-module/loop-step.yaml @@ -0,0 +1,20 @@ +name: loop-step + +report: + final: + - "/tmp/ott/source.txt" + +steps: + - scripts: + - "ExecCmd('mkdir -p /tmp/ott/')" + - "ExecCmd('seq 4 > /tmp/ott/source.txt')" + + - source: '/tmp/ott/source.txt' + threads: '{{ 9 / 2 }}' + commands: + - "echo '---> {{line}} '" + - "sleep 1 && echo '---> done {{line}}'" + + - scripts: + - "ExecCmd('sleep 3')" + - "ExecCmd('echo '<======')" diff --git a/test-workflows/test-module/markdown-generate.yaml b/test-workflows/test-module/markdown-generate.yaml new file mode 100644 index 0000000..34dc94b --- /dev/null +++ b/test-workflows/test-module/markdown-generate.yaml @@ -0,0 +1,15 @@ +name: markdown-generate +desc: HTML report generator based on markdown template + +# go run main.go scan -m test-workflows/test-module/markdown-generate.yaml --debug -t target.io + +report: + final: + - "{{Output}}/subdomain/final-{{Workspace}}.txt" + - "{{Output}}/vuln/active/{{Workspace}}-report.html" + - "{{Output}}/vuln/active/jaeles-summary.txt" + +steps: + - scripts: + - GenMarkdownReport("{{Data}}/markdown/general-template.md", "{{Output}}/summary.html") + - GenMarkdownReport("{{Data}}/markdown/simple-template.md", "{{Output}}/simple.html") diff --git a/test-workflows/test-module/ose.yaml b/test-workflows/test-module/ose.yaml new file mode 100644 index 0000000..f5cfef0 --- /dev/null +++ b/test-workflows/test-module/ose.yaml @@ -0,0 +1,20 @@ +name: test-ose +desc: test-ose + + +steps: + # get data from cdn + - ose: + # run the script directly + - | + if (FileLength('/tmp/ott/sam') > 0) { + ExecCmd('touch /tmp/ott/from-ose && sleep 10') + } + # run the script directly + - | + if (FileLength('/tmp/ott/sam') > 0) { + ExecCmdB('touch /tmp/ott/after-5-ose && sleep 10'); + ExecCmdB('touch /tmp/ott/after-10-ose'); + } + # this will get JS file from ~/osmedeus-plugins/ose/sample.js + - sample.js \ No newline at end of file diff --git a/test-workflows/test-module/s3-cdn.yaml b/test-workflows/test-module/s3-cdn.yaml new file mode 100644 index 0000000..e2f2fed --- /dev/null +++ b/test-workflows/test-module/s3-cdn.yaml @@ -0,0 +1,8 @@ +name: s3-cdn + +steps: + - scripts: + - "ExecCmd('mkdir -p /tmp/ott/')" + - "ExecCmd('seq 10 > /tmp/ott/source.txt')" + - "UploadToS3('/tmp/ott/on-s3.txt')" + - "DownloadFromS3('/tmp/ott/on-s3.txt', '/tmp/on-local-s3.txt')" diff --git a/test-workflows/test-module/set-var.yaml b/test-workflows/test-module/set-var.yaml new file mode 100644 index 0000000..e0da42f --- /dev/null +++ b/test-workflows/test-module/set-var.yaml @@ -0,0 +1,9 @@ +name: set OS Variable + +steps: + + - scripts: + - "SetOSVar('FROM_OSM', 'IS_TRUE')" + - commands: + - echo "=> \$FROM_OSM = $FROM_OSM" + diff --git a/test-workflows/test-module/test-parallel.yaml b/test-workflows/test-module/test-parallel.yaml new file mode 100644 index 0000000..55ce33f --- /dev/null +++ b/test-workflows/test-module/test-parallel.yaml @@ -0,0 +1,18 @@ +name: partest +desc: partest + +params: + - testFile: "/tmp/partest.txt" + - splitLines: "10" + +steps: + - required: + - "{{testFile}}" + scripts: + - SplitFile("{{testFile}}", "{{Workspace}}-index", {{splitLines}}, "/tmp/pt") + + - label: 'Parallel test' + source: "{{testFile}}" + parallel: 1 + commands: + - "echo [[.line]]" \ No newline at end of file diff --git a/test-workflows/test-module/with-threadshold.yaml b/test-workflows/test-module/with-threadshold.yaml new file mode 100644 index 0000000..b44fb9a --- /dev/null +++ b/test-workflows/test-module/with-threadshold.yaml @@ -0,0 +1,14 @@ +name: with-threads-hold + +steps: + - commands: + - "echo 'calc ---> {{ 5555 * threads}} '" + scripts: + - "ExecCmd('mkdir -p /tmp/ott/')" + - "ExecCmd('seq 10 > /tmp/ott/source.txt')" + + - source: '/tmp/ott/source.txt' + threads: '2' + commands: + - "echo '---> {{ 2 * threads}} '" + # - "sleep 1 && echo '---> done [[.line]]'" diff --git a/utils/helper.go b/utils/helper.go new file mode 100644 index 0000000..0ad16fe --- /dev/null +++ b/utils/helper.go @@ -0,0 +1,897 @@ +package utils + +import ( + "archive/zip" + "bufio" + "bytes" + "context" + "crypto/sha1" + "encoding/base64" + "fmt" + "io" + "math/rand" + "net/url" + "os" + "os/exec" + "path" + "path/filepath" + "regexp" + "strconv" + "strings" + + //"syscall" + "text/template" + "time" + + "github.com/mitchellh/go-homedir" +) + +// CalcTimeout calculate timeout +func CalcTimeout(raw string) int { + raw = strings.ToLower(strings.TrimSpace(raw)) + seconds := raw + multiply := 1 + + matched, _ := regexp.MatchString(`.*[a-z]`, raw) + if matched { + unitTime := fmt.Sprintf("%c", raw[len(raw)-1]) + seconds = raw[:len(raw)-1] + switch unitTime { + case "s": + multiply = 1 + break + case "m": + multiply = 60 + break + case "h": + multiply = 3600 + break + } + } + + timeout, err := strconv.Atoi(seconds) + if err != nil { + return 0 + } + return timeout * multiply +} + +// GetDomain get domain from the URL +func GetDomain(raw string) (string, error) { + u, err := url.Parse(raw) + if err == nil { + return u.Hostname(), nil + } + return raw, err +} + +// EmptyDir check if directory is empty or not +func EmptyDir(dir string) bool { + if !FolderExists(dir) { + return true + } + f, err := os.Open(NormalizePath(dir)) + if err != nil { + return false + } + defer f.Close() + + _, err = f.Readdirnames(1) + if err == io.EOF { + return true + } + return false +} + +// EmptyFile check if file is empty or not +func EmptyFile(filename string, num int) bool { + filename = NormalizePath(filename) + if !FileExists(filename) { + return true + } + data := ReadingLines(filename) + if len(data) > num { + return false + } + return true +} + +// StrToInt string to int +func StrToInt(data string) int { + i, err := strconv.Atoi(data) + if err != nil { + return 0 + } + return i +} + +// GetOSEnv get environment variable +func GetOSEnv(name string, defaultValue string) string { + variable, ok := os.LookupEnv(name) + if !ok { + if defaultValue != "" { + return defaultValue + } + return name + } + return variable +} + +// MakeDir just make a folder +func MakeDir(folder string) { + folder = NormalizePath(folder) + os.MkdirAll(folder, 0750) +} + +// GetCurrentDay get current day +func GetCurrentDay() string { + currentTime := time.Now() + return fmt.Sprintf("%v", currentTime.Format("2006-01-02_3:4:5")) +} + +// NormalizePath the path +func NormalizePath(path string) string { + if strings.HasPrefix(path, "~") { + path, _ = homedir.Expand(path) + } + return path +} + +// GetFileContent Reading file and return content of it +func GetFileContent(filename string) string { + var result string + if strings.Contains(filename, "~") { + filename, _ = homedir.Expand(filename) + } + file, err := os.Open(filename) + if err != nil { + return result + } + defer file.Close() + b, err := io.ReadAll(file) + if err != nil { + return result + } + return string(b) +} + +// ReadingLines Reading file and return content as []string +func ReadingLines(filename string) []string { + var result []string + if strings.HasPrefix(filename, "~") { + filename, _ = homedir.Expand(filename) + } + file, err := os.Open(filename) + if err != nil { + return result + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + val := strings.TrimSpace(scanner.Text()) + if val == "" { + continue + } + result = append(result, val) + } + + if err := scanner.Err(); err != nil { + return result + } + return result +} + +// Cat Reading file and return content as []string +func Cat(filename string) { + filename = NormalizePath(filename) + if !FileExists(filename) { + return + } + file, err := os.Open(filename) + if err != nil { + return + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" { + continue + } + fmt.Println(line) + } + return +} + +// ReadingFileUnique Reading file and return content as []string +func ReadingFileUnique(filename string) []string { + var result []string + if strings.Contains(filename, "~") { + filename, _ = homedir.Expand(filename) + } + file, err := os.Open(filename) + if err != nil { + return result + } + defer file.Close() + + seen := make(map[string]bool) + scanner := bufio.NewScanner(file) + for scanner.Scan() { + val := strings.TrimSpace(scanner.Text()) + // unique stuff + if val == "" { + continue + } + if seen[val] { + continue + } + + seen[val] = true + result = append(result, val) + } + + if err := scanner.Err(); err != nil { + return result + } + return result +} + +// WriteToFile write string to a file +func WriteToFile(filename string, data string) (string, error) { + file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + return "", err + } + defer file.Close() + + _, err = io.WriteString(file, data+"\n") + if err != nil { + return "", err + } + return filename, file.Sync() +} + +// AppendToContent append string to a file +func AppendToContent(filename string, data string) (string, error) { + // If the file doesn't exist, create it, or append to the file + f, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return "", err + } + if _, err := f.Write([]byte(data + "\n")); err != nil { + return "", err + } + if err := f.Close(); err != nil { + return "", err + } + return filename, nil +} + +// FileExists check if file is exist or not +func FileExists(filename string) bool { + filename = NormalizePath(filename) + _, err := os.Stat(filename) + if os.IsNotExist(err) { + return false + } + return true +} + +// FolderExists check if file is exist or not +func FolderExists(foldername string) bool { + foldername = NormalizePath(foldername) + if _, err := os.Stat(foldername); os.IsNotExist(err) { + return false + } + return true +} + +// FileLength count len of file +func FileLength(filename string) int { + filename = NormalizePath(filename) + if !FileExists(filename) { + return 0 + } + return CountLines(filename) +} + +// DirLength count len of file +func DirLength(dir string) int { + dir = NormalizePath(dir) + files, err := os.ReadDir(dir) + if err != nil { + dir = dir + "/" + files, err = os.ReadDir(dir) + if err == nil { + return len(files) + } + return 0 + } + return len(files) +} + +// Copy append content to a file +func Copy(src string, dest string) { + src = NormalizePath(src) + dest = NormalizePath(dest) + if !FileExists(src) || FileLength(src) <= 0 { + return + } + input, _ := os.ReadFile(src) + os.WriteFile(dest, input, 0644) +} + +// GetTS get current timestamp and return a string +func GetTS() string { + return strconv.FormatInt(time.Now().Unix(), 10) +} + +// GenHash gen SHA1 hash from string +func GenHash(text string) string { + h := sha1.New() + h.Write([]byte(text)) + hashed := h.Sum(nil) + return fmt.Sprintf("%x", hashed) +} + +// GetFileSize get file size of a file in GB +func GetFileSize(src string) float64 { + var sizeGB float64 + fi, err := os.Stat(NormalizePath(src)) + if err != nil { + return sizeGB + } + // get the size + size := fi.Size() + sizeGB = float64(size) / (1024 * 1024 * 1024) + return sizeGB +} + +// RandomString return a random string with length +func RandomString(n int) string { + var seededRand = rand.New(rand.NewSource(time.Now().UnixNano())) + var letter = []rune("abcdefghijklmnopqrstuvwxyz") + b := make([]rune, n) + for i := range b { + b[i] = letter[seededRand.Intn(len(letter))] + } + return string(b) +} + +// runCmdWithOutput just run os command +func runCmdWithOutput(cmd string) string { + DebugF("Execute: %s", cmd) + command := []string{ + "bash", + "-c", + cmd, + } + realCmd := exec.Command(command[0], command[1:]...) + // output command output to std too + output, _ := realCmd.CombinedOutput() + return string(output) +} + +// RunCmdWithOutput run command with timeout +func RunCmdWithOutput(command string, timeoutRaw ...string) string { + if len(timeoutRaw) == 0 { + return runCmdWithOutput(command) + } + + timeout := CalcTimeout(timeoutRaw[0]) + DebugF("Run command with %v seconds timeout", timeout) + var out string + + c := context.Background() + deadline := time.Now().Add(time.Duration(timeout) * time.Second) + c, cancel := context.WithDeadline(c, deadline) + defer cancel() + go func() { + out = RunCmdWithOutput(command) + cancel() + }() + + select { + case <-c.Done(): + return out + case <-time.After(time.Duration(timeout) * time.Second): + return out + "\n[err] command got timeout" + } +} + +func runCommandWithError(cmd string) (string, error) { + DebugF("Execute: %s", cmd) + command := []string{ + "bash", + "-c", + cmd, + } + var output string + realCmd := exec.Command(command[0], command[1:]...) + + // output command output to std too + cmdReader, _ := realCmd.StdoutPipe() + scanner := bufio.NewScanner(cmdReader) + go func() { + for scanner.Scan() { + out := scanner.Text() + DebugF(out) + output += out + "\n" + } + }() + if err := realCmd.Start(); err != nil { + return output, err + } + if err := realCmd.Wait(); err != nil { + return output, err + } + return output, nil +} + +// RunCommandWithErr Run a command +func RunCommandWithErr(command string, timeoutRaw ...string) (string, error) { + if len(timeoutRaw) == 0 { + return runCommandWithError(command) + } + var output string + var err error + + timeout := CalcTimeout(timeoutRaw[0]) + DebugF("Run command with %v seconds timeout", timeout) + var out string + + c := context.Background() + deadline := time.Now().Add(time.Duration(timeout) * time.Second) + c, cancel := context.WithDeadline(c, deadline) + defer cancel() + go func() { + out, err = runCommandWithError(command) + cancel() + }() + + select { + case <-c.Done(): + return output, err + case <-time.After(time.Duration(timeout) * time.Second): + return out, fmt.Errorf("command got timeout") + } +} + +func RunCommandSteamOutput(cmd string) (string, error) { + DebugF("Execute: %s", cmd) + command := []string{ + "bash", + "-c", + cmd, + } + var output string + realCmd := exec.Command(command[0], command[1:]...) + + // output command output to std too + cmdReader, _ := realCmd.StdoutPipe() + scanner := bufio.NewScanner(cmdReader) + go func() { + for scanner.Scan() { + out := scanner.Text() + fmt.Println(out) + output += out + } + }() + if err := realCmd.Start(); err != nil { + return output, err + } + if err := realCmd.Wait(); err != nil { + return output, err + } + return output, nil +} + +func RunOSCommand(cmd string) (string, error) { + DebugF("Execute: %s", cmd) + command := []string{ + "bash", + "-c", + cmd, + } + var output string + realCmd := exec.Command(command[0], command[1:]...) + + // output command output to std too + cmdReader, _ := realCmd.StdoutPipe() + scanner := bufio.NewScanner(cmdReader) + go func() { + for scanner.Scan() { + out := scanner.Text() + DebugF(out) + output += out + } + }() + if err := realCmd.Start(); err != nil { + return output, err + } + if err := realCmd.Wait(); err != nil { + return output, err + } + return output, nil +} + +// RunCommandWithoutOutput Run a command +func RunCommandWithoutOutput(cmd string) error { + command := []string{ + "bash", + "-c", + cmd, + } + DebugF("[Exec] %v", command) + realCmd := exec.Command(command[0], command[1:]...) + cmdReader, _ := realCmd.StdoutPipe() + scanner := bufio.NewScanner(cmdReader) + go func() { + for scanner.Scan() { + InforF(scanner.Text()) + } + }() + if err := realCmd.Start(); err != nil { + return err + } + if err := realCmd.Wait(); err != nil { + return err + } + return nil +} + +// StripPath just Base64 Encode +func StripPath(raw string) string { + raw = NormalizePath(raw) + raw = strings.Replace(raw, "/", "_", -1) + raw = strings.Replace(raw, "..", "_", -1) + return raw +} + +// ZippedFolder zip a folder +func ZippedFolder(src string, dest string) error { + baseDest := path.Base(dest) + if FileExists(baseDest) { + os.RemoveAll(baseDest) + } + file, err := os.Create(dest + ".zip") + if err != nil { + return err + } + defer file.Close() + + w := zip.NewWriter(file) + defer w.Close() + + walker := func(asbPath string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + file, err := os.Open(asbPath) + if err != nil { + return err + } + defer file.Close() + relPath := strings.Replace(asbPath, src, baseDest, -1) + f, err := w.Create(relPath) + if err != nil { + return err + } + + _, err = io.Copy(f, file) + if err != nil { + return err + } + + return nil + } + err = filepath.Walk(src, walker) + if err != nil { + return err + } + return nil +} + +// CountLines Return the lines amount of the file +func CountLines(filename string) int { + var amount int + if strings.HasPrefix(filename, "~") { + filename, _ = homedir.Expand(filename) + } + file, err := os.Open(filename) + if err != nil { + return amount + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + val := strings.TrimSpace(scanner.Text()) + if val == "" { + continue + } + amount++ + } + + if err := scanner.Err(); err != nil { + return amount + } + return amount +} + +// CleanPath get environment variable +func CleanPath(raw string) string { + var out string + raw = NormalizePath(raw) + base := raw + if FileExists(base) { + base = filepath.Base(raw) + } + + if strings.Count(base, "/") > 2 { + base = base[strings.LastIndex(base, "/")+1:] + if strings.TrimSpace(base) == "" { + domain, err := GetDomain(raw) + if err == nil { + base = domain + } else { + base = RandomString(8) + } + } + } + + out = strings.ReplaceAll(base, "/", "_") + out = strings.ReplaceAll(out, ":", "_") + // DebugF("CleanPath: %s -- %s", raw, out) + return out +} + +func IsFile(src string) bool { + fi, err := os.Stat(NormalizePath(src)) + if err != nil { + return false + } + switch mode := fi.Mode(); { + case mode.IsDir(): + return false + case mode.IsRegular(): + if FileLength(src) > 0 { + return true + } + return false + } + return false +} + +func FolderLength(dir string) int { + dir = NormalizePath(dir) + var length int + if FileExists(dir) { + dir = path.Dir(dir) + } + files, err := os.ReadDir(dir) + if err != nil { + return length + } + length = len(files) + return length +} + +// ImageAsBase64 read image file as a string +func ImageAsBase64(src string) string { + src = NormalizePath(src) + f, err := os.Open(src) + if err != nil { + ErrorF("File not found: %v", src) + return "" + } + + // Read entire JPG into byte slice. + reader := bufio.NewReader(f) + content, err := io.ReadAll(reader) + if err != nil { + return "" + } + // Encode as base64. + encoded := base64.StdEncoding.EncodeToString(content) + return encoded +} + +// Base64Encode read image file as a string +func Base64Encode(raw string) string { + return base64.StdEncoding.EncodeToString([]byte(raw)) +} + +const bufSize = 1024 + +// OffsetRange represents a content block of a file. +type OffsetRange struct { + File string + Start int64 + Stop int64 +} + +// SplitLineChunks splits file into chunks. +// The whole line are guaranteed to be split in the same chunk. +func SplitLineChunks(filename string, chunks int) ([]OffsetRange, error) { + info, err := os.Stat(filename) + if err != nil { + return nil, err + } + + if chunks <= 1 { + return []OffsetRange{ + { + File: filename, + Start: 0, + Stop: info.Size(), + }, + }, nil + } + + file, err := os.Open(filename) + if err != nil { + return nil, err + } + defer file.Close() + + var ranges []OffsetRange + var offset int64 + // avoid the last chunk too few bytes + preferSize := info.Size()/int64(chunks) + 1 + for { + if offset+preferSize >= info.Size() { + ranges = append(ranges, OffsetRange{ + File: filename, + Start: offset, + Stop: info.Size(), + }) + break + } + + offsetRange, err := nextRange(file, offset, offset+preferSize) + if err != nil { + return nil, err + } + + ranges = append(ranges, offsetRange) + if offsetRange.Stop < info.Size() { + offset = offsetRange.Stop + } else { + break + } + } + + return ranges, nil +} + +func nextRange(file *os.File, start, stop int64) (OffsetRange, error) { + offset, err := skipPartialLine(file, stop) + if err != nil { + return OffsetRange{}, err + } + + return OffsetRange{ + File: file.Name(), + Start: start, + Stop: offset, + }, nil +} + +func skipPartialLine(file *os.File, offset int64) (int64, error) { + for { + skipBuf := make([]byte, bufSize) + n, err := file.ReadAt(skipBuf, offset) + if err != nil && err != io.EOF { + return 0, err + } + if n == 0 { + return 0, io.EOF + } + + for i := 0; i < n; i++ { + if skipBuf[i] != '\r' && skipBuf[i] != '\n' { + offset++ + } else { + for ; i < n; i++ { + if skipBuf[i] == '\r' || skipBuf[i] == '\n' { + offset++ + } else { + return offset, nil + } + } + return offset, nil + } + } + } +} + +// A RangeReader is used to read a range of content from a file. +type RangeReader struct { + file *os.File + start int64 + stop int64 +} + +// NewRangeReader returns a RangeReader, which will read the range of content from file. +func NewRangeReader(file *os.File, start, stop int64) *RangeReader { + return &RangeReader{ + file: file, + start: start, + stop: stop, + } +} + +// Read reads the range of content into p. +func (rr *RangeReader) Read(p []byte) (n int, err error) { + stat, err := rr.file.Stat() + if err != nil { + return 0, err + } + + if rr.stop < rr.start || rr.start >= stat.Size() { + return 0, fmt.Errorf("exceed file size") + } + + if rr.stop-rr.start < int64(len(p)) { + p = p[:rr.stop-rr.start] + } + + n, err = rr.file.ReadAt(p, rr.start) + if err != nil { + return n, err + } + + rr.start += int64(n) + return +} + +func Move(src string, dest string) error { + src = NormalizePath(src) + if !IsFile(src) && DirLength(src) == 0 { + return fmt.Errorf("source does not exist: %v", src) + } + + dest = NormalizePath(dest) + os.RemoveAll(dest) + DebugF("Moving %v --> %v", src, dest) + return os.Rename(src, dest) +} + +func IsWritable(filename string) (isWritable bool, err error) { + isWritable = false + info, err := os.Stat(filename) + if err != nil { + return + } + + err = nil + if !info.IsDir() { + return + } + + // Check if the user write bit is enabled in file permission + if info.Mode().Perm()&0200 == 0 { + return + } + + isWritable = true + return +} + +// RenderText resolve template from signature file +func RenderText(format string, data map[string]string) string { + t := template.Must(template.New("").Parse(format)) + buf := &bytes.Buffer{} + err := t.Execute(buf, data) + if err != nil { + return format + } + return buf.String() +} diff --git a/utils/log.go b/utils/log.go new file mode 100644 index 0000000..f88063c --- /dev/null +++ b/utils/log.go @@ -0,0 +1,128 @@ +package utils + +import ( + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/fatih/color" + "github.com/j3ssie/osmedeus/libs" + "github.com/kyokomi/emoji" + "github.com/sirupsen/logrus" + prefixed "github.com/x-cray/logrus-prefixed-formatter" +) + +var logger = logrus.New() + +// InitLog init log +func InitLog(options *libs.Options) { + mwr := io.MultiWriter(os.Stdout) + logDir := libs.LDIR + if options.LogFile == "" { + if !FolderExists(logDir) { + os.MkdirAll(logDir, 0777) + } + tmpFile, err := os.CreateTemp(logDir, "osmedeus-*.log") + if err == nil { + options.LogFile = tmpFile.Name() + } else { + tmpFile, _ := os.CreateTemp("/tmp/", "osmedeus-*.log") + options.LogFile = tmpFile.Name() + } + } + + logDir = filepath.Dir(options.LogFile) + if !FolderExists(logDir) { + os.MkdirAll(logDir, 0777) + } + + f, err := os.OpenFile(options.LogFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) + if err != nil { + fmt.Fprintf(os.Stderr, "error opening log file: %v\n", options.LogFile) + fmt.Fprintf(os.Stderr, "šŸ’” You might want to switch to %v first via %v command", color.HiMagentaString("root user"), color.HiCyanString("sudo su")) + } else { + mwr = io.MultiWriter(os.Stdout, f) + } + + logger = &logrus.Logger{ + Out: mwr, + Level: logrus.InfoLevel, + Formatter: &prefixed.TextFormatter{ + ForceColors: true, + ForceFormatting: true, + FullTimestamp: true, + TimestampFormat: "2006-01-02T15:04:05", + }, + } + + if options.Debug == true { + logger.SetLevel(logrus.DebugLevel) + } else if options.Verbose == true { + logger.SetLevel(logrus.ErrorLevel) + } else if options.Quite == true { + logger.SetOutput(io.Discard) + } +} + +// PrintLine print seperate line +func PrintLine() { + dash := color.HiWhiteString("-") + fmt.Println(strings.Repeat(dash, 40)) +} + +// GoodF print good message +func GoodF(format string, args ...interface{}) { + good := color.HiGreenString("[+]") + fmt.Printf("%s %s\n", good, fmt.Sprintf(format, args...)) +} + +// BannerF print info message +func BannerF(format string, data string) { + banner := fmt.Sprintf("%v%v%v ", color.WhiteString("["), color.BlueString(format), color.WhiteString("]")) + fmt.Printf("%v%v\n", banner, color.HiGreenString(data)) +} + +// BlockF print info message +func BlockF(name string, data string) { + banner := fmt.Sprintf("%v%v%v ", color.WhiteString("["), color.GreenString(name), color.WhiteString("]")) + fmt.Printf(fmt.Sprintf("%v%v\n", banner, data)) +} + +// BadBlockF print info message +func BadBlockF(name string, data string) { + banner := fmt.Sprintf("%v%v%v ", color.WhiteString("["), color.RedString(name), color.WhiteString("]")) + fmt.Printf(fmt.Sprintf("%v%v\n", banner, data)) +} + +// InforF print info message +func InforF(format string, args ...interface{}) { + logger.Info(fmt.Sprintf(format, args...)) +} + +// ErrorF print good message +func ErrorF(format string, args ...interface{}) { + logger.Error(fmt.Sprintf(format, args...)) +} + +// WarnF print good message +func WarnF(format string, args ...interface{}) { + logger.Warning(fmt.Sprintf(format, args...)) +} + +// TraceF print good message +func TraceF(format string, args ...interface{}) { + logger.Trace(fmt.Sprintf(format, args...)) +} + +// DebugF print debug message +func DebugF(format string, args ...interface{}) { + logger.Debug(fmt.Sprintf(format, args...)) +} + +// Emojif print good message +func Emojif(e string, format string, args ...interface{}) string { + emj := strings.TrimSpace(emoji.Sprint(e)) + return fmt.Sprintf("%1s %s", emj, fmt.Sprintf(format, args...)) +} diff --git a/utils/request.go b/utils/request.go new file mode 100644 index 0000000..d8715b3 --- /dev/null +++ b/utils/request.go @@ -0,0 +1,122 @@ +package utils + +import ( + "bytes" + "crypto/tls" + "fmt" + "github.com/j3ssie/osmedeus/libs" + "io" + "net" + "net/http" + "time" +) + +var DefaultClient *http.Client +var ( + UA string +) + +type Response struct { + StatusCode int + Status string + Body string +} + +func InitHTTPClient() { + UA = fmt.Sprintf("Osmedeus/%s by %s", libs.VERSION, libs.AUTHOR) + var transport = &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + DisableKeepAlives: true, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: time.Second, + }).DialContext, + } + + //if options.Proxy != "" { + // proxyUrl, err := url.Parse(options.Proxy) + // if err == nil { + // transport.Proxy = http.ProxyURL(proxyUrl) + // } + //} + + DefaultClient = &http.Client{ + Transport: transport, + } +} + +// SendGET sending GET request +func SendGET(cred string, url string) (res Response) { + DebugF("Sending GET request to: %v", url) + req, err := http.NewRequest("GET", url, nil) + req.Header.Set("User-Agent", UA) + if cred != "" { + req.Header.Set("Authorization", fmt.Sprintf("Basic %s", cred)) + } + resp, err := DefaultClient.Do(req) + + if err != nil { + ErrorF("Error sending to %v - %v", url, err) + return res + } + + defer resp.Body.Close() + resbody, err := io.ReadAll(resp.Body) + if err != nil { + return res + } + res.StatusCode = resp.StatusCode + res.Status = resp.Status + res.Body = string(resbody) + return res +} + +// SendPOST sending POST request +func SendPOST(cred string, url string, body string) (res Response) { + DebugF("Sending POST request to: %v", url) + req, err := http.NewRequest("POST", url, bytes.NewBuffer([]byte(body))) + req.Header.Set("User-Agent", UA) + req.Header.Set("Authorization", fmt.Sprintf("Basic %s", cred)) + req.Header.Set("Content-Type", "application/json") + + resp, err := DefaultClient.Do(req) + if err != nil { + ErrorF("Error sending to %v - %v", url, err) + return res + } + defer resp.Body.Close() + resbody, err := io.ReadAll(resp.Body) + + if err != nil { + return res + } + res.StatusCode = resp.StatusCode + res.Status = resp.Status + res.Body = string(resbody) + return res +} + +// SendPUT sending POST request +func SendPUT(cred string, url string, body string) (res Response) { + DebugF("Sending PUT request to: %v", url) + req, err := http.NewRequest("PUT", url, bytes.NewBuffer([]byte(body))) + req.Header.Set("User-Agent", UA) + req.Header.Set("Authorization", fmt.Sprintf("Basic %s", cred)) + req.Header.Set("Content-Type", "application/json") + + resp, err := DefaultClient.Do(req) + if err != nil { + ErrorF("Error sending to %v - %v", url, err) + return res + } + defer resp.Body.Close() + resbody, err := io.ReadAll(resp.Body) + + if err != nil { + return res + } + res.StatusCode = resp.StatusCode + res.Status = resp.Status + res.Body = string(resbody) + return res +}