From 29f4ad1576033f183f8e35757d09909a1a6ed57a Mon Sep 17 00:00:00 2001 From: sandeep <8293321+ehsandeep@users.noreply.github.com> Date: Mon, 3 Apr 2023 21:07:55 +0530 Subject: [PATCH] initial release --- .github/workflows/build-test.yml | 42 ++++ .github/workflows/codeql-analysis.yml | 41 ++++ .github/workflows/dockerhub-push.yml | 40 ++++ .github/workflows/lint-test.yml | 30 +++ .github/workflows/release-binary.yml | 34 +++ .gitignore | 17 ++ .goreleaser.yml | 38 ++++ Dockerfile | 13 ++ LICENSE | 21 ++ README.md | 245 +++++++++++++++++++++ algo.go | 82 +++++++ cmd/alterx/main.go | 66 ++++++ config.go | 40 ++++ dedupe.go | 61 ++++++ examples/main.go | 20 ++ go.mod | 72 ++++++ go.sum | 242 ++++++++++++++++++++ inputs.go | 93 ++++++++ inputs_test.go | 70 ++++++ internal/dedupe/leveldb.go | 66 ++++++ internal/dedupe/map.go | 30 +++ internal/runner/banner.go | 29 +++ internal/runner/config.go | 49 +++++ internal/runner/runner.go | 131 +++++++++++ mutator.go | 305 ++++++++++++++++++++++++++ mutator_test.go | 49 +++++ permutations.yaml | 167 ++++++++++++++ replacer.go | 27 +++ static/config.yaml | 23 ++ static/domains.txt | 4 + util.go | 61 ++++++ 31 files changed, 2208 insertions(+) create mode 100644 .github/workflows/build-test.yml create mode 100644 .github/workflows/codeql-analysis.yml create mode 100644 .github/workflows/dockerhub-push.yml create mode 100644 .github/workflows/lint-test.yml create mode 100644 .github/workflows/release-binary.yml create mode 100644 .gitignore create mode 100644 .goreleaser.yml create mode 100644 Dockerfile create mode 100644 LICENSE create mode 100644 README.md create mode 100644 algo.go create mode 100644 cmd/alterx/main.go create mode 100644 config.go create mode 100644 dedupe.go create mode 100644 examples/main.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 inputs.go create mode 100644 inputs_test.go create mode 100644 internal/dedupe/leveldb.go create mode 100644 internal/dedupe/map.go create mode 100644 internal/runner/banner.go create mode 100644 internal/runner/config.go create mode 100644 internal/runner/runner.go create mode 100644 mutator.go create mode 100644 mutator_test.go create mode 100644 permutations.yaml create mode 100644 replacer.go create mode 100644 static/config.yaml create mode 100644 static/domains.txt create mode 100644 util.go diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml new file mode 100644 index 00000000..dd68bf9c --- /dev/null +++ b/.github/workflows/build-test.yml @@ -0,0 +1,42 @@ +name: ๐Ÿ”จ Build Test + +on: + workflow_dispatch: + pull_request: + branches: + - dev + paths: + - '**.go' + - '**.mod' +jobs: + build: + name: Test Builds + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macOS-12] + go-version: [1.19.x, 1.20.x] + steps: + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: ${{ matrix.go-version }} + + - name: Check out code + uses: actions/checkout@v3 + + - name: Test + run: go test ./... + working-directory: . + + - name: Build + run: go run . + working-directory: examples/ + + - name: Install + run: go install + working-directory: cmd/alterx/ + + - name: Race Condition Tests + run: echo "www.scanme.sh" | go run -race . + working-directory: cmd/alterx/ diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 00000000..90c913a9 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,41 @@ +name: ๐Ÿšจ CodeQL Analysis + +on: + workflow_dispatch: + pull_request: + branches: + - dev + paths: + - '**.go' + - '**.mod' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'go' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 \ No newline at end of file diff --git a/.github/workflows/dockerhub-push.yml b/.github/workflows/dockerhub-push.yml new file mode 100644 index 00000000..c859f920 --- /dev/null +++ b/.github/workflows/dockerhub-push.yml @@ -0,0 +1,40 @@ +name: ๐ŸŒฅ Docker Push + +on: + workflow_run: + workflows: ["๐ŸŽ‰ Release Binary"] + types: + - completed + workflow_dispatch: + +jobs: + docker: + runs-on: ubuntu-latest-16-cores + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Get Github tag + id: meta + run: | + curl --silent "https://api.github.com/repos/projectdiscovery/alterx/releases/latest" | jq -r .tag_name | xargs -I {} echo TAG={} >> $GITHUB_OUTPUT + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v4 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: projectdiscovery/alterx:latest,projectdiscovery/alterx:${{ steps.meta.outputs.TAG }} \ No newline at end of file diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml new file mode 100644 index 00000000..44158a33 --- /dev/null +++ b/.github/workflows/lint-test.yml @@ -0,0 +1,30 @@ +name: ๐Ÿ™๐Ÿป Lint Test + +on: + workflow_dispatch: + pull_request: + branches: + - dev + paths: + - '**.go' + - '**.mod' +jobs: + lint: + name: Lint Test + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: 1.19 + cache: true + + - name: Run golangci-lint + uses: golangci/golangci-lint-action@v3.4.0 + with: + version: latest + args: --timeout 5m + working-directory: . \ No newline at end of file diff --git a/.github/workflows/release-binary.yml b/.github/workflows/release-binary.yml new file mode 100644 index 00000000..12382a7d --- /dev/null +++ b/.github/workflows/release-binary.yml @@ -0,0 +1,34 @@ +name: ๐ŸŽ‰ Release Binary + +on: + push: + tags: + - v* + workflow_dispatch: + +jobs: + release: + runs-on: ubuntu-latest-16-cores + steps: + - name: "Check out code" + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: "Set up Go" + uses: actions/setup-go@v4 + with: + go-version: 1.19 + cache: true + + - name: "Create release on GitHub" + uses: goreleaser/goreleaser-action@v4 + with: + args: "release --rm-dist" + version: latest + workdir: . + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + SLACK_WEBHOOK: "${{ secrets.RELEASE_SLACK_WEBHOOK }}" + DISCORD_WEBHOOK_ID: "${{ secrets.DISCORD_WEBHOOK_ID }}" + DISCORD_WEBHOOK_TOKEN: "${{ secrets.DISCORD_WEBHOOK_TOKEN }}" \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..d6ee8c52 --- /dev/null +++ b/.gitignore @@ -0,0 +1,17 @@ +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib +cmd/alterx/alterx +dist + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ diff --git a/.goreleaser.yml b/.goreleaser.yml new file mode 100644 index 00000000..4aaf9c42 --- /dev/null +++ b/.goreleaser.yml @@ -0,0 +1,38 @@ +before: + hooks: + - go mod tidy + +builds: +- env: + - CGO_ENABLED=0 + goos: + - windows + - linux + - darwin + goarch: + - amd64 + - 386 + - arm + - arm64 + + binary: '{{ .ProjectName }}' + main: cmd/alterx/main.go + +archives: +- format: zip + replacements: + darwin: macOS + +checksum: + algorithm: sha256 + +announce: + slack: + enabled: true + channel: '#release' + username: GoReleaser + message_template: '{{ .ProjectName }} {{ .Tag }} is out! Check it out at {{ .ReleaseURL }}' + + discord: + enabled: true + message_template: '**New Release: {{ .ProjectName }} {{.Tag}}** is published! Check it out at {{ .ReleaseURL }}' \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..71132ad6 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,13 @@ +FROM golang:1.20.2-alpine AS builder +RUN apk add --no-cache git +WORKDIR /app +COPY . /app +RUN go mod download +RUN go build ./cmd/alterx + +FROM alpine:3.17.2 +RUN apk -U upgrade --no-cache \ + && apk add --no-cache bind-tools ca-certificates +COPY --from=builder /app/alterx /usr/local/bin/ + +ENTRYPOINT ["alterx"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..822f4804 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 ProjectDiscovery + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 00000000..d77a3064 --- /dev/null +++ b/README.md @@ -0,0 +1,245 @@ +

+ AlterX +
+

+ + +

+ + + + + + +

+ +

+ Features โ€ข + Installation โ€ข + Usage โ€ข + Running AlterX โ€ข + Join Discord + +

+ +
+
+   Fast and customizable subdomain wordlist generator using DSL.
+
+
+ +![image](https://user-images.githubusercontent.com/8293321/229380735-140d3f25-d0cb-461d-8c49-4c1eff43d1f4.png) + +## Features +- Fast and Customizable +- **Automatic word enrichment** +- Pre-defined variables +- **Configurable Patterns** +- STDIN / List input + +## Installation +To install alterx, you need to have Golang 1.19 installed on your system. You can download Golang from [here](https://go.dev/doc/install). After installing Golang, you can use the following command to install alterx: + + +```bash +go install github.com/projectdiscovery/alterx/cmd/alterx@latest +``` + +## Help Menu +You can use the following command to see the available flags and options: + +```console +Fast and customizable subdomain wordlist generator using DSL. + +Usage: + ./alterx [flags] + +Flags: +INPUT: + -l, -list string[] subdomains to use when creating permutations (stdin, comma-separated, file) + -p, -pattern string[] custom permutation patterns input to generate (comma-seperated, file) + -pp, -payload value custom payload pattern input to replace/use in key=value format (-pp 'word=words.txt') + +OUTPUT: + -es, -estimate estimate permutation count without generating payloads + -o, -output string output file to write altered subdomain list + -v, -verbose display verbose output + -silent display results only + -version display alterx version + +CONFIG: + -config string alterx cli config file (default '$HOME/.config/alterx/config.yaml') + -en, -enrich enrich wordlist by extracting words from input + -ac string alterx permutation config file (default '$HOME/.config/alterx/permutation_v0.0.1.yaml') + -limit int limit the number of results to return (default 0) + +UPDATE: + -up, -update update alterx to latest version + -duc, -disable-update-check disable automatic alterx update check +``` + +## Why `alterx` ?? + +what makes `alterx` different from any other subdomain permutation tools like `goaltdns` is its `scripting` feature . alterx takes patterns as input and generates subdomain permutation wordlist based on that pattern similar to what [nuclei](https://github.com/projectdiscovery/nuclei) does with [fuzzing-templates](https://github.com/projectdiscovery/fuzzing-templates) . + +What makes `Active Subdomain Enumeration` difficult is the probability of finding a domain that actually exists. If finding possible subdomains is represented on a scale it should look something like + +```console + Using Wordlist < generate permutations with subdomains (goaltdns) < alterx +``` + +Almost all popular subdomain permutation tools have hardcoded patterns and when such tools are run they create wordlist which contain subdomains in Millions and this decreases the feasibility of bruteforcing them with tools like dnsx . There is no actual convention to name subdomains and usually depends on person registering the subdomain. with `alterx` it is possible to create patterns based on results from `passive subdomain enumeration` results which increases probability of finding a subdomain and feasibility to bruteforce them. + +## Variables + +`alterx` uses variable-like syntax similar to nuclei-templates. One can write their own patterns using these variables . when domains are passed as input `alterx` evaluates input and extracts variables from it . + +### Basic / Common Variables + +```yaml +{{sub}} : subdomain prefix or left most part of a subdomain +{{suffix}} : everything except {{sub}} in subdomain name is suffix +{{tld}} : top level domain name (ex com,uk,in etc) +{{etld}} : also know as public suffix (ex co.uk , gov.in etc) +``` + +| Variable | api.scanme.sh | admin.dev.scanme.sh | cloud.scanme.co.uk | +| ---------- | ------------- | ------------------- | ------------------ | +| `{{sub}}` | `api` | `admin` | `cloud` | +| `{{suffix}}` | `scanme.sh` | `dev.scanme.sh` | `scanme.co.uk` | +| `{{tld}}` | `sh` | `sh` | `uk` | +| `{{etld}}` | `-` | `-` | `co.uk` | + +### Advanced Variables + +```yaml +{{root}} : also known as eTLD+1 i.e only root domain (ex for api.scanme.sh => {{root}} is scanme.sh) +{{subN}} : here N is an integer (ex {{sub1}} , {{sub2}} etc) . + +// {{subN}} is advanced variable which exists depending on input +// lets say there is a multi level domain cloud.nuclei.scanme.sh +// in this case {{sub}} = cloud and {{sub1}} = nuclei` +``` + +| Variable | api.scanme.sh | admin.dev.scanme.sh | cloud.scanme.co.uk | +| -------- | ------------- | ------------------- | ------------------ | +| `{{root}}` | `scanme.sh` | `scanme.sh` | `scanme.co.uk` | +| `{{sub1}}` | `-` | `dev` | `-` | +| `{{sub2}}` | `-` | `-` | `-` | + + +## Patterns + +pattern in simple terms can be considered as `template` that describes what type of patterns should alterx generate. + +```console +// Below are some of example patterns which can be used to generate permutations +// assuming api.scanme.sh was given as input and variable {{word}} was given as input with only one value prod +// alterx generates subdomains for below patterns + +"{{sub}}-{{word}}.{{suffix}}" // ex: api-prod.scanme.sh +"{{word}}-{{sub}}.{{suffix}}" // ex: prod-api.scanme.sh +"{{word}}.{{sub}}.{{suffix}}" // ex: prod.api.scanme.sh +"{{sub}}.{{word}}.{{suffix}}" // ex: api.prod.scanme.sh +``` + +Here is an example pattern config file - https://github.com/projectdiscovery/alterx/blob/main/permutations.yaml that can be easily customizable as per need. + +This configuration file generates subdomain permutations for security assessments or penetration tests using customizable patterns and dynamic payloads. Patterns include dash-based, dot-based, and others. Users can create custom payload sections, such as words, region identifiers, or numbers, to suit their specific needs. + +For example, a user could define a new payload section `env` with values like `prod` and `dev`, then use it in patterns like `{{env}}-{{word}}.{{suffix}}` to generate subdomains like `prod-app.example.com` and `dev-api.example.com`. This flexibility allows tailored subdomain list for unique testing scenarios and target environments. + +Default pattern config file used for generation is stored in `$HOME/.config/alterx/` directory, and custom config file can be also used using `-ac` option. + +## Examples + +An example of running alterx on existing list of passive subdomains of `tesla.com` yield us **10 additional NEW** and **valid subdomains** resolved using [dnsx](https://github.com/projectdiscovery/dnsx). + +```console +$ chaos -d tesla.com | alterx | dnsx + + + + ___ ____ _ __ + / _ | / / /____ ____| |/_/ + / __ |/ / __/ -_) __/> < +/_/ |_/_/\__/\__/_/ /_/|_| + + projectdiscovery.io + +[INF] Generated 8312 permutations in 0.0740s +auth-global-stage.tesla.com +auth-stage.tesla.com +digitalassets-stage.tesla.com +errlog-stage.tesla.com +kronos-dev.tesla.com +mfa-stage.tesla.com +paymentrecon-stage.tesla.com +sso-dev.tesla.com +shop-stage.tesla.com +www-uat-dev.tesla.com +``` + +Similarly `-enrich` option can be used to populate known subdomains as world input to generate **target aware permutations**. + +```console +$ chaos -d tesla.com | alterx -enrich + + ___ ____ _ __ + / _ | / / /____ ____| |/_/ + / __ |/ / __/ -_) __/> < +/_/ |_/_/\__/\__/_/ /_/|_| + + projectdiscovery.io + +[INF] Generated 662010 permutations in 3.9989s +``` + +You can alter the default patterns at run time using `-pattern` CLI option. + +```console +$ chaos -d tesla.com | alterx -enrich -p '{{word}}-{{suffix}}' + + ___ ____ _ __ + / _ | / / /____ ____| |/_/ + / __ |/ / __/ -_) __/> < +/_/ |_/_/\__/\__/_/ /_/|_| + + projectdiscovery.io + +[INF] Generated 21523 permutations in 0.7984s +``` + +It is also possible to overwrite existing variables value using `-payload` CLI options. + +```console +$ alterx -list tesla.txt -enrich -p '{{word}}-{{year}}.{{suffix}}' -pp word=keywords.txt -pp year=2023 + + ___ ____ _ __ + / _ | / / /____ ____| |/_/ + / __ |/ / __/ -_) __/> < +/_/ |_/_/\__/\__/_/ /_/|_| + + projectdiscovery.io + +[INF] Generated 21419 permutations in 1.1699s +``` + +**For more information, please checkout the release blog** - https://blog.projectdiscovery.io/introducing-alterx-simplifying-active-subdomain-enumeration-with-patterns/ + + +Do also check out the below similar open-source projects that may fit in your workflow: + +[altdns](https://github.com/infosec-au/altdns), [goaltdns](https://github.com/subfinder/goaltdns), [gotator](https://github.com/Josue87/gotator), [ripgen](https://github.com/resyncgg/ripgen/), [dnsgen](https://github.com/ProjectAnte/dnsgen), [dmut](https://github.com/bp0lr/dmut), [permdns](https://github.com/hpy/permDNS), [str-replace](https://github.com/j3ssie/str-replace), [dnscewl](https://github.com/codingo/DNSCewl), [regulator](https://github.com/cramppet/regulator) + + +-------- + +
+ +**alterx** is made with โค๏ธ by the [projectdiscovery](https://projectdiscovery.io) team and distributed under [MIT License](LICENSE.md). + + +Join Discord + +
\ No newline at end of file diff --git a/algo.go b/algo.go new file mode 100644 index 00000000..22291cf9 --- /dev/null +++ b/algo.go @@ -0,0 +1,82 @@ +package alterx + +// Nth Order ClusterBomb with variable length array/values +func ClusterBomb(payloads *IndexMap, callback func(varMap map[string]interface{}), Vector []string) { + // The Goal of implementation is to reduce number of arbitary values by constructing a vector + + // Algorithm + // step 1) Initialize/Input a IndexMap(Here: payloads) + // indexMap is nothing but a map with all of keys indexed in a different map + + // step 2) Vector is n length array such that n = len(payloads) + // Each value in payloads(IndexMap) contains a array + // ex: payloads["word"] = []string{"api","dev","cloud"} + + // step 3) Initial length of Vector is 0 . By using recursion + // we construct a Vector with all possible values of payloads[N] where N = 0 < len(payloads) + + // step 4) At end of recursion len(Vector) == len(payloads).Cap() - 1 + // which translates that Vn = {r0,r1,...,rn} and only rn is missing + // in this case/situation iterate over all possible values of rn i.e payload.GetNth(n) + if len(Vector) == payloads.Cap()-1 { + // end of vector + vectorMap := map[string]interface{}{} + for k, v := range Vector { + // construct a map[variable]=value with all available vectors + vectorMap[payloads.KeyAtNth(k)] = v + } + // one element a.k.a last element is missing from ^ map + index := len(Vector) + for _, elem := range payloads.GetNth(index) { + vectorMap[payloads.KeyAtNth(index)] = elem + callback(vectorMap) + } + return + } + + // step 5) if vector is not filled until payload.Cap()-1 + // iterate over rth variable payloads and execute them using recursion + // if Vector is empty or at 1st index fix iterate over xth position + index := len(Vector) + for _, v := range payloads.GetNth(index) { + tmp := []string{} + if len(Vector) > 0 { + tmp = append(tmp, Vector...) + } + tmp = append(tmp, v) + ClusterBomb(payloads, callback, tmp) // Recursion + } +} + +type IndexMap struct { + values map[string][]string + indexes map[int]string +} + +func (o *IndexMap) GetNth(n int) []string { + return o.values[o.indexes[n]] +} + +func (o *IndexMap) Cap() int { + return len(o.values) +} + +// KeyAtNth returns key present at Nth position +func (o *IndexMap) KeyAtNth(n int) string { + return o.indexes[n] +} + +// NewIndexMap returns type such that elements of map can be retrieved by a fixed index +func NewIndexMap(values map[string][]string) *IndexMap { + i := &IndexMap{ + values: values, + } + indexes := map[int]string{} + counter := 0 + for k := range values { + indexes[counter] = k + counter++ + } + i.indexes = indexes + return i +} diff --git a/cmd/alterx/main.go b/cmd/alterx/main.go new file mode 100644 index 00000000..6dc1f461 --- /dev/null +++ b/cmd/alterx/main.go @@ -0,0 +1,66 @@ +package main + +import ( + "io" + "os" + + "github.com/projectdiscovery/alterx" + "github.com/projectdiscovery/alterx/internal/runner" + "github.com/projectdiscovery/gologger" +) + +func main() { + + cliOpts := runner.ParseFlags() + + alterOpts := alterx.Options{ + Domains: cliOpts.Domains, + Patterns: cliOpts.Patterns, + Payloads: cliOpts.Payloads, + Limit: cliOpts.Limit, + Enrich: cliOpts.Enrich, // enrich payloads + } + + if cliOpts.PermutationConfig != "" { + // read config + config, err := alterx.NewConfig(cliOpts.PermutationConfig) + if err != nil { + gologger.Fatal().Msgf("failed to read %v file got: %v", cliOpts.PermutationConfig, err) + } + if len(config.Patterns) > 0 { + alterOpts.Patterns = config.Patterns + } + if len(config.Payloads) > 0 { + alterOpts.Payloads = config.Payloads + } + } + + // configure output writer + var output io.Writer + if cliOpts.Output != "" { + fs, err := os.OpenFile(cliOpts.Output, os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + gologger.Fatal().Msgf("failed to open output file %v got %v", cliOpts.Output, err) + } + output = fs + defer fs.Close() + } else { + output = os.Stdout + } + + // create new alterx instance with options + m, err := alterx.New(&alterOpts) + if err != nil { + gologger.Fatal().Msgf("failed to parse alterx config got %v", err) + } + + if cliOpts.Estimate { + gologger.Info().Msgf("Estimated Payloads (including duplicates) : %v", m.EstimateCount()) + return + } + + if err = m.ExecuteWithWriter(output); err != nil { + gologger.Error().Msgf("failed to write output to file got %v", err) + } + +} diff --git a/config.go b/config.go new file mode 100644 index 00000000..3bed36ad --- /dev/null +++ b/config.go @@ -0,0 +1,40 @@ +package alterx + +import ( + "os" + + _ "embed" + + "github.com/projectdiscovery/gologger" + "gopkg.in/yaml.v3" +) + +//go:embed permutations.yaml +var DefaultPermutationsBin []byte + +// DefaultConfig contains default patterns and payloads +var DefaultConfig Config + +type Config struct { + Patterns []string `yaml:"patterns"` + Payloads map[string][]string `yaml:"payloads"` +} + +// NewConfig reads config from file +func NewConfig(filePath string) (*Config, error) { + bin, err := os.ReadFile(filePath) + if err != nil { + return nil, err + } + var cfg Config + if err = yaml.Unmarshal(bin, &cfg); err != nil { + return nil, err + } + return &cfg, nil +} + +func init() { + if err := yaml.Unmarshal(DefaultPermutationsBin, &DefaultConfig); err != nil { + gologger.Error().Msgf("default wordlist not found: got %v", err) + } +} diff --git a/dedupe.go b/dedupe.go new file mode 100644 index 00000000..3c301110 --- /dev/null +++ b/dedupe.go @@ -0,0 +1,61 @@ +package alterx + +import "github.com/projectdiscovery/alterx/internal/dedupe" + +// MaxInMemoryDedupeSize (default : 100 MB) +var MaxInMemoryDedupeSize = 100 * 1024 * 1024 + +type DedupeBackend interface { + // Upsert add/update key to backend/database + Upsert(elem string) + // Execute given callback on each element while iterating + IterCallback(callback func(elem string)) + // Cleanup cleans any residuals after deduping + Cleanup() +} + +// Dedupe is string deduplication type which removes +// all duplicates if +type Dedupe struct { + receive <-chan string + backend DedupeBackend +} + +// Drains channel and tries to dedupe it +func (d *Dedupe) Drain() { + for { + val, ok := <-d.receive + if !ok { + break + } + d.backend.Upsert(val) + } +} + +// GetResults iterates over dedupe storage and returns results +func (d *Dedupe) GetResults() <-chan string { + send := make(chan string, 100) + go func() { + defer close(send) + d.backend.IterCallback(func(elem string) { + send <- elem + }) + d.backend.Cleanup() + }() + return send +} + +// NewDedupe returns a dedupe instance which removes all duplicates +// Note: If byteLen is not correct/specified alterx may consume lot of memory +func NewDedupe(ch <-chan string, byteLen int) *Dedupe { + d := &Dedupe{ + receive: ch, + } + if byteLen <= MaxInMemoryDedupeSize { + d.backend = dedupe.NewMapBackend() + } else { + // gologger print a info message here + d.backend = dedupe.NewLevelDBBackend() + } + return d +} diff --git a/examples/main.go b/examples/main.go new file mode 100644 index 00000000..567708f1 --- /dev/null +++ b/examples/main.go @@ -0,0 +1,20 @@ +package main + +import ( + "os" + + "github.com/projectdiscovery/alterx" + "github.com/projectdiscovery/gologger" +) + +func main() { + opts := &alterx.Options{ + Domains: []string{"api.scanme.sh", "chaos.scanme.sh", "nuclei.scanme.sh", "cloud.nuclei.scanme.sh"}, + } + + m, err := alterx.New(opts) + if err != nil { + gologger.Fatal().Msg(err.Error()) + } + m.ExecuteWithWriter(os.Stdout) +} diff --git a/go.mod b/go.mod new file mode 100644 index 00000000..9b9f9066 --- /dev/null +++ b/go.mod @@ -0,0 +1,72 @@ +module github.com/projectdiscovery/alterx + +go 1.19 + +require ( + github.com/projectdiscovery/fasttemplate v0.0.2 + github.com/projectdiscovery/goflags v0.1.8 + github.com/projectdiscovery/gologger v1.1.8 + github.com/projectdiscovery/utils v0.0.17 + github.com/stretchr/testify v1.8.2 + github.com/syndtr/goleveldb v1.0.0 + golang.org/x/net v0.8.0 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + aead.dev/minisign v0.2.0 // indirect + github.com/Masterminds/semver/v3 v3.2.0 // indirect + github.com/VividCortex/ewma v1.2.0 // indirect + github.com/alecthomas/chroma v0.10.0 // indirect + github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect + github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect + github.com/aymerick/douceur v0.2.0 // indirect + github.com/charmbracelet/glamour v0.6.0 // indirect + github.com/cheggaaa/pb/v3 v3.1.2 // indirect + github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dlclark/regexp2 v1.8.1 // indirect + github.com/dsnet/compress v0.0.1 // indirect + github.com/fatih/color v1.14.1 // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/snappy v0.0.1 // indirect + github.com/google/go-github/v30 v30.1.0 // indirect + github.com/google/go-querystring v1.1.0 // indirect + github.com/gorilla/css v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/logrusorgru/aurora v2.0.3+incompatible // indirect + github.com/lucasb-eyer/go-colorful v1.2.0 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.17 // indirect + github.com/mattn/go-runewidth v0.0.14 // indirect + github.com/mholt/archiver v3.1.1+incompatible // indirect + github.com/microcosm-cc/bluemonday v1.0.23 // indirect + github.com/miekg/dns v1.1.50 // indirect + github.com/minio/selfupdate v0.6.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/muesli/reflow v0.3.0 // indirect + github.com/muesli/termenv v0.15.1 // indirect + github.com/nwaples/rardecode v1.1.0 // indirect + github.com/olekukonko/tablewriter v0.0.5 // indirect + github.com/pierrec/lz4 v2.6.0+incompatible // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rivo/uniseg v0.4.4 // indirect + github.com/rogpeppe/go-internal v1.10.0 // indirect + github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect + github.com/ulikunitz/xz v0.5.8 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect + github.com/yuin/goldmark v1.5.4 // indirect + github.com/yuin/goldmark-emoji v1.0.1 // indirect + golang.org/x/crypto v0.7.0 // indirect + golang.org/x/exp v0.0.0-20221019170559-20944726eadf // indirect + golang.org/x/mod v0.8.0 // indirect + golang.org/x/oauth2 v0.5.0 // indirect + golang.org/x/sys v0.6.0 // indirect + golang.org/x/tools v0.6.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.28.1 // indirect + gopkg.in/djherbis/times.v1 v1.3.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 00000000..1681f1a6 --- /dev/null +++ b/go.sum @@ -0,0 +1,242 @@ +aead.dev/minisign v0.2.0 h1:kAWrq/hBRu4AARY6AlciO83xhNnW9UaC8YipS2uhLPk= +aead.dev/minisign v0.2.0/go.mod h1:zdq6LdSd9TbuSxchxwhpA9zEb9YXcVGoE8JakuiGaIQ= +github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= +github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= +github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= +github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= +github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbfjek= +github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s= +github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= +github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/aymanbagabas/go-osc52 v1.0.3/go.mod h1:zT8H+Rk4VSabYN90pWyugflM3ZhpTZNC7cASDfUCdT4= +github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= +github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= +github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= +github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= +github.com/charmbracelet/glamour v0.6.0 h1:wi8fse3Y7nfcabbbDuwolqTqMQPMnVPeZhDM273bISc= +github.com/charmbracelet/glamour v0.6.0/go.mod h1:taqWV4swIMMbWALc0m7AfE9JkPSU8om2538k9ITBxOc= +github.com/cheggaaa/pb/v3 v3.1.2 h1:FIxT3ZjOj9XJl0U4o2XbEhjFfZl7jCVCDOGq1ZAB7wQ= +github.com/cheggaaa/pb/v3 v3.1.2/go.mod h1:SNjnd0yKcW+kw0brSusraeDd5Bf1zBfxAzTL2ss3yQ4= +github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ= +github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= +github.com/dlclark/regexp2 v1.8.1 h1:6Lcdwya6GjPUNsBct8Lg/yRPwMhABj269AAzdGSiR+0= +github.com/dlclark/regexp2 v1.8.1/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= +github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= +github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= +github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= +github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= +github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY= +github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-github/v30 v30.1.0 h1:VLDx+UolQICEOKu2m4uAoMti1SxuEBAl7RSEG16L+Oo= +github.com/google/go-github/v30 v30.1.0/go.mod h1:n8jBpHl45a/rlBUtRJMOG4GhNADUQFEufcolZ95JfU8= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= +github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= +github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= +github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= +github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= +github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mholt/archiver v3.1.1+incompatible h1:1dCVxuqs0dJseYEhi5pl7MYPH9zDa1wBi7mF09cbNkU= +github.com/mholt/archiver v3.1.1+incompatible/go.mod h1:Dh2dOXnSdiLxRiPoVfIr/fI1TwETms9B8CTWfeh7ROU= +github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM= +github.com/microcosm-cc/bluemonday v1.0.23 h1:SMZe2IGa0NuHvnVNAZ+6B38gsTbi5e4sViiWJyDDqFY= +github.com/microcosm-cc/bluemonday v1.0.23/go.mod h1:mN70sk7UkkF8TUr2IGBpNN0jAgStuPzlK76QuruE/z4= +github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= +github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= +github.com/minio/selfupdate v0.6.0 h1:i76PgT0K5xO9+hjzKcacQtO7+MjJ4JKA8Ak8XQ9DDwU= +github.com/minio/selfupdate v0.6.0/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= +github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= +github.com/muesli/termenv v0.13.0/go.mod h1:sP1+uffeLaEYpyOTb8pLCUctGcGLnoFjSn4YJK5e2bc= +github.com/muesli/termenv v0.15.1 h1:UzuTb/+hhlBugQz28rpzey4ZuKcZ03MeKsoG7IJZIxs= +github.com/muesli/termenv v0.15.1/go.mod h1:HeAQPTzpfs016yGtA4g00CsdYnVLJvxsS4ANqrZs2sQ= +github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7MQ= +github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= +github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/pierrec/lz4 v2.6.0+incompatible h1:Ix9yFKn1nSPBLFl/yZknTp8TU5G4Ps0JDmguYK6iH1A= +github.com/pierrec/lz4 v2.6.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA= +github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw= +github.com/projectdiscovery/goflags v0.1.8 h1:Urhm2Isq2BdRt8h4h062lHKYXO65RHRjGTDSkUwex/g= +github.com/projectdiscovery/goflags v0.1.8/go.mod h1:Yxi9tclgwGczzDU65ntrwaIql5cXeTvW5j2WxFuF+Jk= +github.com/projectdiscovery/gologger v1.1.8 h1:CFlCzGlqAhPqWIrAXBt1OVh5jkMs1qgoR/z4xhdzLNE= +github.com/projectdiscovery/gologger v1.1.8/go.mod h1:bNyVaC1U/NpJtFkJltcesn01NR3K8Hg6RsLVce6yvrw= +github.com/projectdiscovery/utils v0.0.17 h1:Y/uj8wAI1/a9UtWwDTBCBgsc2RicLUhrhcbCCsC7nrM= +github.com/projectdiscovery/utils v0.0.17/go.mod h1:Cu216AlQ7rAYa8aDBqB2OgNfu5p24Uj+tG9RxV8Wbfs= +github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= +github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= +github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= +github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= +github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= +github.com/ulikunitz/xz v0.5.7/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ulikunitz/xz v0.5.8 h1:ERv8V6GKqVi23rgu5cj9pVfVzJbOqAY2Ntl88O6c2nQ= +github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= +github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.5.2/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yuin/goldmark v1.5.4 h1:2uY/xC0roWy8IBEGLgB1ywIoEJFGmRrX21YQcvGZzjU= +github.com/yuin/goldmark v1.5.4/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yuin/goldmark-emoji v1.0.1 h1:ctuWEyzGBwiucEqxzwe0SOYDXPAucOrE9NQC18Wa1os= +github.com/yuin/goldmark-emoji v1.0.1/go.mod h1:2w1E6FEWLcDQkoTE+7HU6QF1F6SLlNGjRIBbIZQFqkQ= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= +golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/exp v0.0.0-20221019170559-20944726eadf h1:nFVjjKDgNY37+ZSYCJmtYf7tOlfQswHqplG2eosjOMg= +golang.org/x/exp v0.0.0-20221019170559-20944726eadf/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.5.0 h1:HuArIo48skDwlrvM3sEdHXElYslAMsf3KwRkkW4MC4s= +golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210228012217-479acdf4ea46/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/djherbis/times.v1 v1.3.0 h1:uxMS4iMtH6Pwsxog094W0FYldiNnfY/xba00vq6C2+o= +gopkg.in/djherbis/times.v1 v1.3.0/go.mod h1:AQlg6unIsrsCEdQYhTzERy542dz6SFdQFZFv6mUY0P8= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/inputs.go b/inputs.go new file mode 100644 index 00000000..e8d8bb6e --- /dev/null +++ b/inputs.go @@ -0,0 +1,93 @@ +package alterx + +import ( + "fmt" + "strconv" + "strings" + + "github.com/projectdiscovery/gologger" + urlutil "github.com/projectdiscovery/utils/url" + "golang.org/x/net/publicsuffix" +) + +// Input contains parsed/evaluated data of a URL +type Input struct { + TLD string // only TLD (right most part of subdomain) ex: `.uk` + ETLD string // Simply put public suffix (ex: co.uk) + Root string // Root Domain (eTLD+1) of Subdomain + Sub string // Sub or LeftMost prefix of subdomain + Suffix string // suffix is everything except `Sub` (Note: if domain is not multilevel Suffix==Root) + MultiLevel []string // (Optional) store prefix of multi level subdomains +} + +// GetMap returns variables map of input +func (i *Input) GetMap() map[string]interface{} { + m := map[string]interface{}{ + "tld": i.TLD, + "etld": i.ETLD, + "root": i.Root, + "sub": i.Sub, + "suffix": i.Suffix, + } + for k, v := range i.MultiLevel { + m["sub"+strconv.Itoa(k+1)] = v + } + for k, v := range m { + if v == "" { + // purge empty vars + delete(m, k) + } + } + return m +} + +// NewInput parses URL to Input Vars +func NewInput(inputURL string) (*Input, error) { + URL, err := urlutil.Parse(inputURL) + if err != nil { + return nil, err + } + // check if hostname contains * + if strings.Contains(URL.Hostname(), "*") { + if strings.HasPrefix(URL.Hostname(), "*.") { + tmp := strings.TrimPrefix(URL.Hostname(), "*.") + URL.Host = strings.Replace(URL.Host, URL.Hostname(), tmp, 1) + } + // if * is present in middle ex: prod.*.hackerone.com + // skip it + if strings.Contains(URL.Hostname(), "*") { + return nil, fmt.Errorf("input %v is not a valid url , skipping", inputURL) + } + } + ivar := &Input{} + suffix, _ := publicsuffix.PublicSuffix(URL.Hostname()) + if strings.Contains(suffix, ".") { + ivar.ETLD = suffix + arr := strings.Split(suffix, ".") + ivar.TLD = arr[len(arr)-1] + } else { + ivar.TLD = suffix + } + rootDomain, err := publicsuffix.EffectiveTLDPlusOne(URL.Hostname()) + if err != nil { + // this happens if input domain does not have eTLD+1 at all ex: `.com` or `co.uk` + gologger.Warning().Msgf("input domain %v is eTLD/publicsuffix and not a valid domain name", URL.Hostname()) + return ivar, nil + } + ivar.Root = rootDomain + // anything before root domain is subdomain + subdomainPrefix := strings.TrimSuffix(URL.Hostname(), rootDomain) + subdomainPrefix = strings.TrimSuffix(subdomainPrefix, ".") + if strings.Contains(subdomainPrefix, ".") { + // this is a multi level subdomain + // ex: something.level.scanme.sh + // in such cases variable name starts after 1st prefix + prefixes := strings.Split(subdomainPrefix, ".") + ivar.Sub = prefixes[0] + ivar.MultiLevel = prefixes[1:] + } else { + ivar.Sub = subdomainPrefix + } + ivar.Suffix = strings.TrimPrefix(URL.Hostname(), ivar.Sub+".") + return ivar, nil +} diff --git a/inputs_test.go b/inputs_test.go new file mode 100644 index 00000000..56277105 --- /dev/null +++ b/inputs_test.go @@ -0,0 +1,70 @@ +package alterx + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestInput(t *testing.T) { + testcases := []string{"scanme.co.uk", "https://scanme.co.uk", "scanme.co.uk:443", "https://scanme.co.uk:443"} + expected := &Input{ + TLD: "uk", + ETLD: "co.uk", + Root: "scanme.co.uk", + Suffix: "scanme.co.uk", + Sub: "", + } + for _, v := range testcases { + got, err := NewInput(v) + require.Nilf(t, err, "failed to parse url %v", v) + require.Equal(t, expected, got) + } +} + +func TestInputSub(t *testing.T) { + testcases := []struct { + url string + expected *Input + }{ + {url: "something.scanme.sh", expected: &Input{TLD: "sh", ETLD: "", Root: "scanme.sh", Sub: "something", Suffix: "scanme.sh"}}, + {url: "nested.something.scanme.sh", expected: &Input{TLD: "sh", ETLD: "", Root: "scanme.sh", Sub: "nested", Suffix: "something.scanme.sh", MultiLevel: []string{"something"}}}, + {url: "nested.multilevel.scanme.co.uk", expected: &Input{TLD: "uk", ETLD: "co.uk", Root: "scanme.co.uk", Sub: "nested", Suffix: "multilevel.scanme.co.uk", MultiLevel: []string{"multilevel"}}}, + {url: "sub.level1.level2.scanme.sh", expected: &Input{TLD: "sh", ETLD: "", Root: "scanme.sh", Sub: "sub", Suffix: "level1.level2.scanme.sh", MultiLevel: []string{"level1", "level2"}}}, + {url: "scanme.sh", expected: &Input{TLD: "sh", ETLD: "", Sub: "", Suffix: "scanme.sh", Root: "scanme.sh"}}, + } + for _, v := range testcases { + got, err := NewInput(v.url) + require.Nilf(t, err, "failed to parse url %v", v.url) + require.Equal(t, v.expected, got, *v.expected) + } +} + +func TestVarCount(t *testing.T) { + testcases := []struct { + statement string + count int + }{ + {statement: "{{sub}}.something.{{tld}}", count: 2}, + {statement: "{{sub}}.{{sub1}}.{{sub2}}.{{root}}", count: 4}, + {statement: "no variables", count: 0}, + } + for _, v := range testcases { + require.EqualValues(t, v.count, getVarCount(v.statement), "variable count mismatch") + } +} + +func TestExtractVar(t *testing.T) { + // extract all variables from statement + testcases := []struct { + statement string + expected []string + }{ + {statement: "{{sub}}.something.{{tld}}", expected: []string{"sub", "tld"}}, + {statement: "{{sub}}.{{sub1}}.{{sub2}}.{{root}}", expected: []string{"sub", "sub1", "sub2", "root"}}, + {statement: "no variables", expected: []string{}}, + } + for _, v := range testcases { + require.Equal(t, v.expected, getAllVars(v.statement)) + } +} diff --git a/internal/dedupe/leveldb.go b/internal/dedupe/leveldb.go new file mode 100644 index 00000000..f6060a3a --- /dev/null +++ b/internal/dedupe/leveldb.go @@ -0,0 +1,66 @@ +package dedupe + +import ( + "os" + "reflect" + "unsafe" + + "github.com/projectdiscovery/gologger" + "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/errors" +) + +type LevelDBBackend struct { + storage *leveldb.DB + tempdir string +} + +func NewLevelDBBackend() *LevelDBBackend { + l := &LevelDBBackend{} + dbPath, err := os.MkdirTemp("", "nuclei-report-*") + if err != nil { + gologger.Fatal().Msgf("failed to create temp dir for alterx dedupe got: %v", err) + } + l.tempdir = dbPath + l.storage, err = leveldb.OpenFile(dbPath, nil) + if err != nil { + if !errors.IsCorrupted(err) { + gologger.Fatal().Msgf("goleveldb: failed to open db got %v", err) + } + // If the metadata is corrupted, try to recover + l.storage, err = leveldb.RecoverFile(dbPath, nil) + if err != nil { + gologger.Fatal().Msgf("goleveldb: corrupted db found, recovery failed got %v", err) + } + } + return l +} + +func (l *LevelDBBackend) Upsert(elem string) { + if err := l.storage.Put(unsafeToBytes(elem), nil, nil); err != nil { + gologger.Error().Msgf("dedupe: leveldb: got %v while writing %v", err, elem) + } +} + +func (l *LevelDBBackend) IterCallback(callback func(elem string)) { + iter := l.storage.NewIterator(nil, nil) + for iter.Next() { + callback(string(iter.Key())) + } +} + +func (l *LevelDBBackend) Cleanup() { + if err := os.RemoveAll(l.tempdir); err != nil { + gologger.Error().Msgf("leveldb: cleanup got %v", err) + } +} + +// unsafeToBytes converts a string to byte slice and does it with +// zero allocations. +// +// Reference - https://stackoverflow.com/questions/59209493/how-to-use-unsafe-get-a-byte-slice-from-a-string-without-memory-copy +func unsafeToBytes(data string) []byte { + var buf = *(*[]byte)(unsafe.Pointer(&data)) + (*reflect.SliceHeader)(unsafe.Pointer(&buf)).Cap = len(data) + return buf +} diff --git a/internal/dedupe/map.go b/internal/dedupe/map.go new file mode 100644 index 00000000..94f8edb8 --- /dev/null +++ b/internal/dedupe/map.go @@ -0,0 +1,30 @@ +package dedupe + +import "runtime/debug" + +type MapBackend struct { + storage map[string]struct{} +} + +func NewMapBackend() *MapBackend { + return &MapBackend{storage: map[string]struct{}{}} +} + +func (m *MapBackend) Upsert(elem string) { + m.storage[elem] = struct{}{} +} + +func (m *MapBackend) IterCallback(callback func(elem string)) { + for k := range m.storage { + callback(k) + } +} + +func (m *MapBackend) Cleanup() { + m.storage = nil + // By default GC doesnot release buffered/allocated memory + // since there always is possibilitly of needing it again/immediately + // and releases memory in chunks + // debug.FreeOSMemory forces GC to release allocated memory at once + debug.FreeOSMemory() +} diff --git a/internal/runner/banner.go b/internal/runner/banner.go new file mode 100644 index 00000000..fa624278 --- /dev/null +++ b/internal/runner/banner.go @@ -0,0 +1,29 @@ +package runner + +import ( + "github.com/projectdiscovery/gologger" + updateutils "github.com/projectdiscovery/utils/update" +) + +var banner = (` + ___ ____ _ __ + / _ | / / /____ ____| |/_/ + / __ |/ / __/ -_) __/> < +/_/ |_/_/\__/\__/_/ /_/|_| +`) + +var version = "v0.0.1" + +// showBanner is used to show the banner to the user +func showBanner() { + gologger.Print().Msgf("%s\n", banner) + gologger.Print().Msgf("\t\tprojectdiscovery.io\n\n") +} + +// GetUpdateCallback returns a callback function that updates katana +func GetUpdateCallback() func() { + return func() { + showBanner() + updateutils.GetUpdateToolCallback("alterx", version)() + } +} diff --git a/internal/runner/config.go b/internal/runner/config.go new file mode 100644 index 00000000..f2395e13 --- /dev/null +++ b/internal/runner/config.go @@ -0,0 +1,49 @@ +package runner + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/projectdiscovery/alterx" + "github.com/projectdiscovery/gologger" + fileutil "github.com/projectdiscovery/utils/file" + "gopkg.in/yaml.v3" +) + +func getUserHomeDir() string { + homeDir, err := os.UserHomeDir() + if err != nil { + panic(err) + } + return homeDir +} + +func init() { + defaultPermutationCfg := filepath.Join(getUserHomeDir(), fmt.Sprintf(".config/alterx/permutation_%v.yaml", version)) + // create default permutation.yaml config if does not exist + if fileutil.FileExists(defaultPermutationCfg) { + // if it exists use that data as default + if bin, err := os.ReadFile(defaultPermutationCfg); err == nil { + var cfg alterx.Config + if errx := yaml.Unmarshal(bin, &cfg); errx == nil { + alterx.DefaultConfig = cfg + return + } + } + } + if err := validateDir(filepath.Join(getUserHomeDir(), ".config/alterx")); err != nil { + gologger.Error().Msgf("alterx config dir not found and failed to create got: %v", err) + } + if err := os.WriteFile(defaultPermutationCfg, alterx.DefaultPermutationsBin, 0600); err != nil { + gologger.Error().Msgf("failed to save default config to %v got: %v", defaultPermutationCfg, err) + } +} + +// validateDir checks if dir exists if not creates it +func validateDir(dirPath string) error { + if fileutil.FolderExists(dirPath) { + return nil + } + return fileutil.CreateFolder(dirPath) +} diff --git a/internal/runner/runner.go b/internal/runner/runner.go new file mode 100644 index 00000000..76fce42c --- /dev/null +++ b/internal/runner/runner.go @@ -0,0 +1,131 @@ +package runner + +import ( + "fmt" + "io" + "os" + "strings" + + "github.com/projectdiscovery/goflags" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/gologger/levels" + fileutil "github.com/projectdiscovery/utils/file" + updateutils "github.com/projectdiscovery/utils/update" +) + +type Options struct { + Domains goflags.StringSlice // Subdomains to use as base + Patterns goflags.StringSlice // Input Patterns + Payloads map[string][]string // Input Payloads/WordLists + Output string + Config string + PermutationConfig string + Estimate bool + DisableUpdateCheck bool + Verbose bool + Silent bool + Enrich bool + Limit int + // internal/unexported fields + wordlists goflags.RuntimeMap +} + +func ParseFlags() *Options { + opts := &Options{} + flagSet := goflags.NewFlagSet() + flagSet.SetDescription(`Fast and customizable subdomain wordlist generator using DSL.`) + + flagSet.CreateGroup("input", "Input", + flagSet.StringSliceVarP(&opts.Domains, "list", "l", nil, "subdomains to use when creating permutations (stdin, comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions), + flagSet.StringSliceVarP(&opts.Patterns, "pattern", "p", nil, "custom permutation patterns input to generate (comma-seperated, file)", goflags.FileCommaSeparatedStringSliceOptions), + flagSet.RuntimeMapVarP(&opts.wordlists, "payload", "pp", nil, "custom payload pattern input to replace/use in key=value format (-pp 'word=words.txt')"), + ) + + flagSet.CreateGroup("output", "Output", + flagSet.BoolVarP(&opts.Estimate, "estimate", "es", false, "estimate permutation count without generating payloads"), + flagSet.StringVarP(&opts.Output, "output", "o", "", "output file to write altered subdomain list"), + flagSet.BoolVarP(&opts.Verbose, "verbose", "v", false, "display verbose output"), + flagSet.BoolVar(&opts.Silent, "silent", false, "display results only"), + flagSet.CallbackVar(printVersion, "version", "display alterx version"), + ) + + flagSet.CreateGroup("config", "Config", + flagSet.StringVar(&opts.Config, "config", "", `alterx cli config file (default '$HOME/.config/alterx/config.yaml')`), + flagSet.BoolVarP(&opts.Enrich, "enrich", "en", false, "enrich wordlist by extracting words from input"), + flagSet.StringVar(&opts.PermutationConfig, "ac", "", fmt.Sprintf(`alterx permutation config file (default '$HOME/.config/alterx/permutation_%v.yaml')`, version)), + flagSet.IntVar(&opts.Limit, "limit", 0, "limit the number of results to return (default 0)"), + ) + + flagSet.CreateGroup("update", "Update", + flagSet.CallbackVarP(GetUpdateCallback(), "update", "up", "update alterx to latest version"), + flagSet.BoolVarP(&opts.DisableUpdateCheck, "disable-update-check", "duc", false, "disable automatic alterx update check"), + ) + + if err := flagSet.Parse(); err != nil { + gologger.Fatal().Msgf("Could not read flags: %s\n", err) + } + + if opts.Config != "" { + if err := flagSet.MergeConfigFile(opts.Config); err != nil { + gologger.Error().Msgf("failed to read config file got %v", err) + } + } + + if opts.Silent { + gologger.DefaultLogger.SetMaxLevel(levels.LevelSilent) + } else if opts.Verbose { + gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose) + } + showBanner() + + if !opts.DisableUpdateCheck { + latestVersion, err := updateutils.GetVersionCheckCallback("alterx")() + if err != nil { + if opts.Verbose { + gologger.Error().Msgf("alterx version check failed: %v", err.Error()) + } + } else { + gologger.Info().Msgf("Current alterx version %v %v", version, updateutils.GetVersionDescription(version, latestVersion)) + } + } + + opts.Payloads = map[string][]string{} + for k, v := range opts.wordlists.AsMap() { + value, ok := v.(string) + if !ok { + continue + } + if fileutil.FileExists(value) { + bin, err := os.ReadFile(value) + if err != nil { + gologger.Error().Msgf("failed to read wordlist %v got %v", value, err) + continue + } + wordlist := strings.Fields(string(bin)) + opts.Payloads[k] = wordlist + } else { + opts.Payloads[k] = []string{value} + } + } + + // read from stdin + if fileutil.HasStdin() { + bin, err := io.ReadAll(os.Stdin) + if err != nil { + gologger.Error().Msgf("failed to read input from stdin got %v", err) + } + opts.Domains = strings.Fields(string(bin)) + } + + // TODO: replace Options.Domains with Input String Channel + if len(opts.Domains) == 0 { + gologger.Fatal().Msgf("alterx: no input found") + } + + return opts +} + +func printVersion() { + gologger.Info().Msgf("Current version: %s", version) + os.Exit(0) +} diff --git a/mutator.go b/mutator.go new file mode 100644 index 00000000..feb23567 --- /dev/null +++ b/mutator.go @@ -0,0 +1,305 @@ +package alterx + +import ( + "bytes" + "context" + "fmt" + "io" + "regexp" + "strings" + "time" + + "github.com/projectdiscovery/fasttemplate" + "github.com/projectdiscovery/gologger" + errorutil "github.com/projectdiscovery/utils/errors" + sliceutil "github.com/projectdiscovery/utils/slice" +) + +var ( + extractNumbers = regexp.MustCompile(`[0-9]+`) + extractWords = regexp.MustCompile(`[a-zA-Z0-9]+`) + extractWordsOnly = regexp.MustCompile(`[a-zA-Z]{3,}`) + DedupeResults = true // Dedupe all results (default: true) +) + +// Mutator Options +type Options struct { + // list of Domains to use as base + Domains []string + // list of words to use while creating permutations + // if empty DefaultWordList is used + Payloads map[string][]string + // list of pattersn to use while creating permutations + // if empty DefaultPatterns are used + Patterns []string + // Limits output results (0 = no limit) + Limit int + // Enrich when true alterx extra possible words from input + // and adds them to default payloads word,number + Enrich bool +} + +// Mutator +type Mutator struct { + Options *Options + payloadCount int + Inputs []*Input // all processed inputs + timeTaken time.Duration + // internal or unexported variables + maxkeyLenInBytes int +} + +// New creates and returns new mutator instance from options +func New(opts *Options) (*Mutator, error) { + if len(opts.Domains) == 0 { + return nil, fmt.Errorf("no input provided to calculate permutations") + } + if len(opts.Payloads) == 0 { + opts.Payloads = map[string][]string{} + if len(DefaultConfig.Payloads) == 0 { + return nil, fmt.Errorf("something went wrong, `DefaultWordList` and input wordlist are empty") + } + opts.Payloads = DefaultConfig.Payloads + } + if len(opts.Patterns) == 0 { + if len(DefaultConfig.Patterns) == 0 { + return nil, fmt.Errorf("something went wrong,`DefaultPatters` and input patterns are empty") + } + opts.Patterns = DefaultConfig.Patterns + } + // purge duplicates if any + for k, v := range opts.Payloads { + dedupe := sliceutil.Dedupe(v) + if len(v) != len(dedupe) { + gologger.Warning().Msgf("%v duplicate payloads found in %v. purging them..", len(v)-len(dedupe), k) + opts.Payloads[k] = dedupe + } + } + m := &Mutator{ + Options: opts, + } + if err := m.validatePatterns(); err != nil { + return nil, err + } + if err := m.prepareInputs(); err != nil { + return nil, err + } + if opts.Enrich { + m.enrichPayloads() + } + return m, nil +} + +// Execute calculates all permutations using input wordlist and patterns +// and writes them to a string channel +func (m *Mutator) Execute(ctx context.Context) <-chan string { + var maxBytes int + if DedupeResults { + count := m.EstimateCount() + maxBytes = count * m.maxkeyLenInBytes + } + + results := make(chan string, len(m.Options.Patterns)) + go func() { + now := time.Now() + for _, v := range m.Inputs { + varMap := getSampleMap(v.GetMap(), m.Options.Payloads) + for _, pattern := range m.Options.Patterns { + if err := checkMissing(pattern, varMap); err == nil { + statement := Replace(pattern, v.GetMap()) + select { + case <-ctx.Done(): + return + default: + m.clusterBomb(statement, results) + } + } else { + gologger.Warning().Msgf("%v : failed to evaluate pattern %v. skipping", err.Error(), pattern) + } + } + } + m.timeTaken = time.Since(now) + close(results) + }() + + if DedupeResults { + // drain results + d := NewDedupe(results, maxBytes) + d.Drain() + return d.GetResults() + } + return results +} + +// ExecuteWithWriter executes Mutator and writes results directly to type that implements io.Writer interface +func (m *Mutator) ExecuteWithWriter(Writer io.Writer) error { + if Writer == nil { + return errorutil.NewWithTag("alterx", "writer destination cannot be nil") + } + resChan := m.Execute(context.TODO()) + m.payloadCount = 0 + for { + value, ok := <-resChan + if !ok { + gologger.Info().Msgf("Generated %v permutations in %v", m.payloadCount, m.Time()) + return nil + } + if m.Options.Limit > 0 && m.payloadCount == m.Options.Limit { + gologger.Info().Msgf("Generated %v permutations in %v", m.payloadCount, m.Time()) + return nil + } + _, err := Writer.Write([]byte(value + "\n")) + m.payloadCount++ + if err != nil { + return err + } + } +} + +// EstimateCount estimates number of payloads that will be created +// without actually executing/creating permutations +func (m *Mutator) EstimateCount() int { + counter := 0 + for _, v := range m.Inputs { + varMap := getSampleMap(v.GetMap(), m.Options.Payloads) + for _, pattern := range m.Options.Patterns { + if err := checkMissing(pattern, varMap); err == nil { + // if say patterns is {{sub}}.{{sub1}}-{{word}}.{{root}} + // and input domain is api.scanme.sh its clear that {{sub1}} here will be empty/missing + // in such cases `alterx` silently skips that pattern for that specific input + // this way user can have a long list of patterns but they are only used if all required data is given (much like self-contained templates) + statement := Replace(pattern, v.GetMap()) + bin := unsafeToBytes(statement) + if m.maxkeyLenInBytes < len(bin) { + m.maxkeyLenInBytes = len(bin) + } + varsUsed := getAllVars(statement) + if len(varsUsed) == 0 { + counter += 1 + } else { + tmpCounter := 1 + for _, word := range varsUsed { + tmpCounter *= len(m.Options.Payloads[word]) + } + counter += tmpCounter + } + } + } + } + return counter +} + +// DryRun executes payloads without storing and returns number of payloads created +// this value is also stored in variable and can be accessed via getter `PayloadCount` +func (m *Mutator) DryRun() int { + m.payloadCount = 0 + err := m.ExecuteWithWriter(io.Discard) + if err != nil { + gologger.Error().Msgf("alterx: got %v", err) + } + return m.payloadCount +} + +// clusterBomb calculates all payloads of clusterbomb attack and sends them to result channel +func (m *Mutator) clusterBomb(template string, results chan string) { + // Early Exit: this is what saves clusterBomb from stackoverflows and reduces + // n*len(n) iterations and n recursions + varsUsed := getAllVars(template) + if len(varsUsed) == 0 { + // clusterBomb is not required + // just send existing template as result and exit + results <- template + return + } + payloadSet := map[string][]string{} + // instead of sending all payloads only send payloads that are used + // in template/statement + for _, v := range varsUsed { + payloadSet[v] = []string{} + for _, word := range m.Options.Payloads[v] { + if !strings.Contains(template, word) { + // skip all words that are already present in template/sub , it is highly unlikely + // we will ever find api-api.example.com + payloadSet[v] = append(payloadSet[v], word) + } + } + } + payloads := NewIndexMap(payloadSet) + // in clusterBomb attack no of payloads generated are + // len(first_set)*len(second_set)*len(third_set).... + callbackFunc := func(varMap map[string]interface{}) { + results <- Replace(template, varMap) + } + ClusterBomb(payloads, callbackFunc, []string{}) +} + +// prepares input and patterns and calculates estimations +func (m *Mutator) prepareInputs() error { + errors := []string{} + // prepare input + allInputs := []*Input{} + for _, v := range m.Options.Domains { + i, err := NewInput(v) + if err != nil { + errors = append(errors, err.Error()) + continue + } + allInputs = append(allInputs, i) + } + m.Inputs = allInputs + if len(errors) > 0 { + gologger.Warning().Msgf("errors found when preparing inputs got: %v : skipping errored inputs", strings.Join(errors, " : ")) + } + return nil +} + +// validates all patterns by compiling them +func (m *Mutator) validatePatterns() error { + for _, v := range m.Options.Patterns { + // check if all placeholders are correctly used and are valid + if _, err := fasttemplate.NewTemplate(v, ParenthesisOpen, ParenthesisClose); err != nil { + return err + } + } + return nil +} + +// enrichPayloads extract possible words and adds them to default wordlist +func (m *Mutator) enrichPayloads() { + var temp bytes.Buffer + for _, v := range m.Inputs { + temp.WriteString(v.Sub + " ") + if len(v.MultiLevel) > 0 { + temp.WriteString(strings.Join(v.MultiLevel, " ")) + } + } + numbers := extractNumbers.FindAllString(temp.String(), -1) + extraWords := extractWords.FindAllString(temp.String(), -1) + extraWordsOnly := extractWordsOnly.FindAllString(temp.String(), -1) + if len(extraWordsOnly) > 0 { + extraWords = append(extraWords, extraWordsOnly...) + extraWords = sliceutil.Dedupe(extraWords) + } + + if len(m.Options.Payloads["word"]) > 0 { + extraWords = append(extraWords, m.Options.Payloads["word"]...) + m.Options.Payloads["word"] = sliceutil.Dedupe(extraWords) + } + if len(m.Options.Payloads["number"]) > 0 { + numbers = append(numbers, m.Options.Payloads["number"]...) + m.Options.Payloads["number"] = sliceutil.Dedupe(numbers) + } +} + +// PayloadCount returns total estimated payloads count +func (m *Mutator) PayloadCount() int { + if m.payloadCount == 0 { + return m.EstimateCount() + } + return m.payloadCount +} + +// Time returns time taken to create permutations in seconds +func (m *Mutator) Time() string { + return fmt.Sprintf("%.4fs", m.timeTaken.Seconds()) +} diff --git a/mutator_test.go b/mutator_test.go new file mode 100644 index 00000000..b62fb44c --- /dev/null +++ b/mutator_test.go @@ -0,0 +1,49 @@ +package alterx + +import ( + "bytes" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +var testConfig = Config{ + Patterns: []string{ + "{{sub}}-{{word}}.{{root}}", // ex: api-prod.scanme.sh + "{{word}}-{{sub}}.{{root}}", // ex: prod-api.scanme.sh + "{{word}}.{{sub}}.{{root}}", // ex: prod.api.scanme.sh + "{{sub}}.{{word}}.{{root}}", // ex: api.prod.scanme.sh + }, + Payloads: map[string][]string{ + "word": {"dev", "lib", "prod", "stage", "wp"}, + }, +} + +func TestMutatorCount(t *testing.T) { + opts := &Options{ + Domains: []string{"api.scanme.sh", "chaos.scanme.sh", "nuclei.scanme.sh", "cloud.nuclei.scanme.sh"}, + } + opts.Patterns = testConfig.Patterns + opts.Payloads = testConfig.Payloads + + expectedCount := len(opts.Patterns) * len(opts.Payloads["word"]) * len(opts.Domains) + m, err := New(opts) + require.Nil(t, err) + require.EqualValues(t, expectedCount, m.EstimateCount()) +} + +func TestMutatorResults(t *testing.T) { + opts := &Options{ + Domains: []string{"api.scanme.sh", "chaos.scanme.sh", "nuclei.scanme.sh", "cloud.nuclei.scanme.sh"}, + } + opts.Patterns = testConfig.Patterns + opts.Payloads = testConfig.Payloads + m, err := New(opts) + require.Nil(t, err) + var buff bytes.Buffer + err = m.ExecuteWithWriter(&buff) + require.Nil(t, err) + count := strings.Split(strings.TrimSpace(buff.String()), "\n") + require.EqualValues(t, 80, len(count), buff.String()) +} diff --git a/permutations.yaml b/permutations.yaml new file mode 100644 index 00000000..6ccb417d --- /dev/null +++ b/permutations.yaml @@ -0,0 +1,167 @@ +patterns: + # Dash based patterns ex: api-dev.scanme.sh + - "{{word}}-{{sub}}.{{suffix}}" # dev-api + - "{{sub}}-{{word}}.{{suffix}}" # api-dev + # Dot based patterns ex: dev.api.scanme.sh + - "{{word}}.{{sub}}.{{suffix}}" + - "{{sub}}.{{word}}.{{suffix}}" + # Iteration based + - "{{sub}}{{number}}.{{suffix}}" + # replace current subdomain name + - "{{word}}.{{suffix}}" + # No Seperator (ex: devtest.scanme.sh) + - "{{sub}}{{word}}.{{suffix}}" + # add region perfix + - "{{region}}.{{sub}}.{{suffix}}" + # clusterbomb words and numbers + # - "{{word}}{{number}}.{{suffix}}" + +## Note: +# `-enrich/-e` option adds new words to `word` and `number` payloads only +payloads: + word: + - "api" + - "k8s" + - "v1" + - "v2" + - "origin" + - "raw" + - "stage" + - "test" + - "qa" + - "web" + - "prod" + - "service" + - "grafana" + - "beta" + - "admin" + - "staging" + - "wordpress" + - "wp" + - "dev" + - "app" + - "mta-sts" + - "tech" + - "private" + - "public" + - "login" + - "role" + - "backend" + - "cloud" + - "internal" + - "mail" + - "oauth" + - "oauth2" + - "vpn" + - "lab" + - "local" + - "live" + - "data" + - "mobile" + - "search" + - "stats" + - "final" + - "ldap" + - "media" + - "docs" + - "eng" + - "engineering" + - "market" + - "compute" + - "cdn" + - "acc" + - "access" + - "backup" + - "blogs" + - "blog" + - "careers" + - "client" + - "cms" + - "cms1" + - "conf" + - "dmz" + - "drupal" + - "corp" + - "faq" + - "ir" + - "legacy" + - "log" + - "logs" + - "dashboard" + - "monitor" + - "mysql" + - "mssql" + - "db" + - "partner" + - "payment" + - "pay" + - "office" + - "plugins" + - "shop" + - "prometheus" + - "stripe" + - "forum" + - "manager" + - "server" + - "core" + - "content" + - "ads" + - "shopify" + - "o1" + - "s1" + - "s3" + - "promotion" + - "temp" + - "my" + - "proxy" + - "asset" + - "assets" + - "atlas" + - "build" + - "builds" + - "code" + - "info" + - "image" + - "review" + - "developers" + - "developer" + - "administrator" + - "www" + - "www1" + - "www2" + - "netlify" + - "storage" + + region: + - "us-east-1" + - "us-east-2" + - "us-west-1" + - "us-west-2" + - "eu-east-1" + - "eu-central-1" + + number: + - "1" + - "2" + - "3" + - "4" + - "5" + - "6" + - "7" + - "8" + - "9" + - "10" + - "18" + - "20" + - "21" + - "22" + - "23" + - "24" + - "2017" + - "2018" + - "2019" + - "2023" + - "2024" + - "2022" + - "2021" + - "2020" \ No newline at end of file diff --git a/replacer.go b/replacer.go new file mode 100644 index 00000000..2807078e --- /dev/null +++ b/replacer.go @@ -0,0 +1,27 @@ +package alterx + +import ( + "fmt" + + "github.com/projectdiscovery/fasttemplate" +) + +const ( + // General marker (open/close) + General = "ยง" + // ParenthesisOpen marker - begin of a placeholder + ParenthesisOpen = "{{" + // ParenthesisClose marker - end of a placeholder + ParenthesisClose = "}}" +) + +// Replace replaces placeholders in template with values on the fly. +func Replace(template string, values map[string]interface{}) string { + valuesMap := make(map[string]interface{}, len(values)) + for k, v := range values { + valuesMap[k] = fmt.Sprint(v) + } + replaced := fasttemplate.ExecuteStringStd(template, ParenthesisOpen, ParenthesisClose, valuesMap) + final := fasttemplate.ExecuteStringStd(replaced, General, General, valuesMap) + return final +} diff --git a/static/config.yaml b/static/config.yaml new file mode 100644 index 00000000..ecc79299 --- /dev/null +++ b/static/config.yaml @@ -0,0 +1,23 @@ + +patterns: + - '{{sub}}-{{word}}.{{suffix}}' + - '{{word}}-{{sub}}.{{suffix}}' + - '{{word}}.{{sub}}.{{suffix}}' + - '{{sub}}.{{word}}.{{suffix}}' + ## If required data is not available pattern is silently discarded + ## for that specific template + - '{{sub}}{{year}}.{{suffix}}' + - '{{sub}}-{{word}}{{year}}.{{suffix}}' +payloads: + word: + - dev + - lib + - prod + - stage + - wp + year: + - "2021" + - "2022" + - "2023" + - "22" + - "23" diff --git a/static/domains.txt b/static/domains.txt new file mode 100644 index 00000000..82e67a0e --- /dev/null +++ b/static/domains.txt @@ -0,0 +1,4 @@ +api.scanme.sh +chaos.scanme.sh +nuclei.scanme.sh +cloud.nuclei.scanme.sh \ No newline at end of file diff --git a/util.go b/util.go new file mode 100644 index 00000000..21abaef0 --- /dev/null +++ b/util.go @@ -0,0 +1,61 @@ +package alterx + +import ( + "fmt" + "reflect" + "regexp" + "strings" + "unsafe" +) + +var varRegex = regexp.MustCompile(`\{\{([a-zA-Z0-9]+)\}\}`) + +// returns no of variables present in statement +func getVarCount(data string) int { + return len(varRegex.FindAllStringSubmatch(data, -1)) +} + +// returns names of all variables +func getAllVars(data string) []string { + values := []string{} + for _, v := range varRegex.FindAllStringSubmatch(data, -1) { + if len(v) >= 2 { + values = append(values, v[1]) + } + } + return values +} + +// getSampleMap returns a sample map containing input variables and payload variable +func getSampleMap(inputVars map[string]interface{}, payloadVars map[string][]string) map[string]interface{} { + sMap := map[string]interface{}{} + for k, v := range inputVars { + sMap[k] = v + } + for k, v := range payloadVars { + if k != "" && len(v) > 0 { + sMap[k] = "temp" + } + } + return sMap +} + +// checkMissing checks if all variables/placeholders are successfully replaced +// if not error is thrown with description +func checkMissing(template string, data map[string]interface{}) error { + got := Replace(template, data) + if res := varRegex.FindAllString(got, -1); len(res) > 0 { + return fmt.Errorf("values of `%v` variables not found", strings.Join(res, ",")) + } + return nil +} + +// unsafeToBytes converts a string to byte slice and does it with +// zero allocations. +// +// Reference - https://stackoverflow.com/questions/59209493/how-to-use-unsafe-get-a-byte-slice-from-a-string-without-memory-copy +func unsafeToBytes(data string) []byte { + var buf = *(*[]byte)(unsafe.Pointer(&data)) + (*reflect.SliceHeader)(unsafe.Pointer(&buf)).Cap = len(data) + return buf +}