Merge pull request #58 from projectdiscovery/dev

v0.0.9

.github/workflows/build-test.yml

@@ -16,7 +16,7 @@ jobs:
           go-version: 1.18
 
       - name: Check out code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Test
         run: go test .

.github/workflows/codeql-analysis.yml

@@ -23,7 +23,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

.github/workflows/dockerhub-push.yml

@@ -1,34 +1,39 @@
 name: 🌥 Docker Push
 
 on:
-  release:
-    types: [published]
+  workflow_run:
+    workflows: ["🎉 Release Binary"]
+    types:
+      - completed
   workflow_dispatch:
 
 jobs:
   docker:
     runs-on: ubuntu-latest
     steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v2
-      -
-        name: Set up QEMU
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Get Github tag
+        id: meta
+        run: |
+          echo "::set-output name=tag::$(curl --silent "https://api.github.com/repos/projectdiscovery/mapcidr/releases/latest" | jq -r .tag_name)"
+      - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
-      -
-        name: Set up Docker Buildx
+
+      - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
-      -
-        name: Login to DockerHub
+
+      - name: Login to DockerHub
         uses: docker/login-action@v1 
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_TOKEN }}
-      -
-        name: Build and push
+
+      - name: Build and push
         uses: docker/build-push-action@v2
         with:
           context: .
           platforms: linux/amd64,linux/arm64,linux/arm
           push: true
-          tags: projectdiscovery/mapcidr:latest
+          tags: projectdiscovery/mapcidr:latest,projectdiscovery/mapcidr:${{ steps.meta.outputs.tag }}

.github/workflows/lint-test.yml

@@ -9,10 +9,14 @@ jobs:
     name: Lint Test
     runs-on: ubuntu-latest
     steps:
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.17
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       - name: Run golangci-lint
-        uses: golangci/golangci-lint-action@v2
+        uses: golangci/golangci-lint-action@v3.1.0
         with:
           version: latest
           args: --timeout 5m

README.md

@@ -43,42 +43,49 @@ mapCIDR is developed to ease load distribution for mass scanning operations, it
 # Installation
 
 ```sh
-▶ go install -v github.com/projectdiscovery/mapcidr/cmd/mapcidr@latest
+go install -v github.com/projectdiscovery/mapcidr/cmd/mapcidr@latest
 ```
 
 # Usage
 
 ```sh
-▶ mapcidr -h
+mapcidr -h
 ```
 
 This will display help for the tool. Here are all the switches it supports.
 
-| Flag             | Description                                        | Example                            |
-|------------------|----------------------------------------------------|------------------------------------|
-| `-aggregate`     | Aggregate CIDRs into the minimum number            | `mapcidr -aggregate`               |
-| `-cidr`          | Single CIDR to process                             | `mapcidr -cidr 173.0.84.0/24`      |
-| `-ips`           | File containing ips to process                     | `mapcidr -ips ips.txt`             |
-| `-sbc`           | Slice by CIDR count                                | `mapcidr -sbc 10`                  |
-| `-sbh`           | Slice by HOST count                                | `mapcidr -sbh 10000`               |
-| `-l`             | File containing list of CIDRs                      | `mapcidr -l cidr.txt`              |
-| `-o`             | File to write output to (optional)                 | `mapcidr -o output.txt`            |
-| `-silent`        | Make the output silent                             | `mapcidr -silent`                  |
-| `-version`       | Print current version of `mapcidr` client          | `mapcidr -version`                 |
-| `-shuffle`       | Shuffle IP with masscan blackrock cipher           | `mapcidr -shuffle`                 |
-| `-shuffle-ports` | Shuffle IP:port with comma separated list of ports | `mapcidr -shuffle-ports 21,80,443` |
+```yaml
+INPUT:
+   -cidr string          CIDR to process
+   -l, -list string      File containing list of CIDRs to process
+   -il, -ip-list string  File containing list of IPs to process
+
+PROCESS:
+   -sbc int                   Slice CIDRs by given CIDR count
+   -sbh int                   Slice CIDRs by given HOST count
+   -agg, -aggregate           Aggregate IPs/CIDRs into the minimum subnet
+   -sip, -shuffle-ip          Shuffle input ip
+   -sp, -shuffle-port string  Shuffle input ip:port
+
+OUTPUT:
+   -o, -output string  File to write output to
+   -silent             Silent mode
+   -version            Show version
+   -skip-base          Skip base IPs (ending in .0) in output
+   -skip-broadcast     Skip broadcast IPs (ending in .255) in output
+```
 
 # Running mapCIDR
 
 In order to get list of IPs for a give CIDR, use the following command.
 
 ### CIDR expansion
 
-```sh
-▶ mapcidr -cidr 173.0.84.0/24
+```console
+mapcidr -cidr 173.0.84.0/24
 ```
 
-```sh
+```console
                    ____________  ___    
   __ _  ___ ____  / ___/  _/ _ \/ _ \
  /  ' \/ _ '/ _ \/ /___/ // // / , _/   
@@ -107,11 +114,11 @@ In order to get list of IPs for a give CIDR, use the following command.
 In order to slice given CIDR or list of CIDR by CIDR count or slice into multiple and equal smaller subnets, use the following command.
 
 
-```sh
-▶ mapcidr -cidr 173.0.84.0/24 -sbc 10 -silent
+```console
+mapcidr -cidr 173.0.84.0/24 -sbc 10 -silent
 ```
 
-```
+```console
 173.0.84.0/27
 173.0.84.32/27
 173.0.84.64/27
@@ -128,11 +135,11 @@ In order to slice given CIDR or list of CIDR by CIDR count or slice into multipl
 
 In order to slice given CIDR for equal number of host count in each CIDR, use the following command.
 
-```sh
-▶ mapcidr -cidr 173.0.84.0/16 -sbh 20000 -silent
+```console
+mapcidr -cidr 173.0.84.0/16 -sbh 20000 -silent
 ```
 
-```
+```console
 173.0.0.0/18
 173.0.64.0/18
 173.0.128.0/18
@@ -145,22 +152,22 @@ Note: it's possible to obtain a perfect split only when the desired amount of sl
 
 In order to merge multiple CIDR ranges into smaller subnet block, use the following command.
 
-```sh
-▶ mapcidr -l cidrs.txt -silent -aggregate
+```console
+mapcidr -l cidrs.txt -silent -aggregate
 ```
 
 In order to list CIDR blocks for given list of IPs, use the following command.
 
-```sh
-▶ mapcidr -l ips.txt -silent -aggregate
+```console
+mapcidr -l ips.txt -silent -aggregate
 ```
 
 ### CIDR based IP filtering
 
 In order to filter IPs from the given list of CIDR ranges, use the following command.
 
-```sh
-▶ mapcidr -ips ip-list.txt -l cirds.txt
+```console
+mapcidr -ips ip-list.txt -l cirds.txt
 ```
 
 # Use mapCIDR as a library

cmd/mapcidr/main.go

@@ -2,14 +2,14 @@ package main
 
 import (
 	"bufio"
-	"flag"
 	"net"
 	"os"
 	"strconv"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/projectdiscovery/goflags"
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/gologger/levels"
 	"github.com/projectdiscovery/ipranger"
@@ -18,17 +18,19 @@ import (
 
 // Options contains cli options
 type Options struct {
-	FileIps      string
-	Slices       int
-	HostCount    int
-	Cidr         string
-	FileCidr     string
-	Silent       bool
-	Version      bool
-	Output       string
-	Aggregate    bool
-	Shuffle      bool
-	ShufflePorts string
+	FileIps         string
+	Slices          int
+	HostCount       int
+	Cidr            string
+	FileCidr        string
+	Silent          bool
+	Version         bool
+	Output          string
+	Aggregate       bool
+	Shuffle         bool
+	ShufflePorts    string
+	SkipBaseIP      bool
+	SkipBroadcastIP bool
 	// NoColor   bool
 	// Verbose   bool
 }
@@ -37,12 +39,12 @@ const banner = `
                    ____________  ___    
   __ _  ___ ____  / ___/  _/ _ \/ _ \   
  /  ' \/ _ '/ _ \/ /___/ // // / , _/   
-/_/_/_/\_,_/ .__/\___/___/____/_/|_| v0.0.8
+/_/_/_/\_,_/ .__/\___/___/____/_/|_| v0.0.9
           /_/                                                     	 
 `
 
 // Version is the current version of mapcidr
-const Version = `v0.0.7`
+const Version = `v0.0.9`
 
 // showBanner is used to show the banner to the user
 func showBanner() {
@@ -56,19 +58,35 @@ func showBanner() {
 // ParseOptions parses the command line options for application
 func ParseOptions() *Options {
 	options := &Options{}
+	flagSet := goflags.NewFlagSet()
+	flagSet.SetDescription(`mapCIDR is developed to ease load distribution for mass scanning operations, it can be used both as a library and as independent CLI tool.`)
+
+	//input
+	createGroup(flagSet, "input", "Input",
+		flagSet.StringVar(&options.Cidr, "cidr", "", "CIDR to process"),
+		flagSet.StringVarP(&options.FileCidr, "list", "l", "", "File containing list of CIDRs to process"),
+		flagSet.StringVarP(&options.FileIps, "ip-list", "il", "", "File containing list of IPs to process"),
+	)
+
+	//Process
+	createGroup(flagSet, "process", "Process",
+		flagSet.IntVar(&options.Slices, "sbc", 0, "Slice CIDRs by given CIDR count"),
+		flagSet.IntVar(&options.HostCount, "sbh", 0, "Slice CIDRs by given HOST count"),
+		flagSet.BoolVarP(&options.Aggregate, "aggregate", "agg", false, "Aggregate IPs/CIDRs into the minimum subnet"),
+		flagSet.BoolVarP(&options.Shuffle, "shuffle-ip", "sip", false, "Shuffle input ip"),
+		flagSet.StringVarP(&options.ShufflePorts, "shuffle-port", "sp", "", "Shuffle input ip:port"),
+	)
 
-	flag.StringVar(&options.FileIps, "ips", "", "File containing ips to process")
-	flag.BoolVar(&options.Aggregate, "aggregate", false, "Aggregate CIDRs into the minimum number")
-	flag.IntVar(&options.Slices, "sbc", 0, "Slice by CIDR count")
-	flag.IntVar(&options.HostCount, "sbh", 0, "Slice by HOST count")
-	flag.StringVar(&options.Cidr, "cidr", "", "Single CIDR to process")
-	flag.StringVar(&options.FileCidr, "l", "", "File containing CIDR")
-	flag.StringVar(&options.Output, "o", "", "File to write output to (optional)")
-	flag.BoolVar(&options.Silent, "silent", false, "Silent mode")
-	flag.BoolVar(&options.Shuffle, "shuffle", false, "Shuffle Ips")
-	flag.StringVar(&options.ShufflePorts, "shuffle-ports", "", "Shuffle Ips with ports")
-	flag.BoolVar(&options.Version, "version", false, "Show version")
-	flag.Parse()
+	//output
+	createGroup(flagSet, "output", "Output",
+		flagSet.StringVarP(&options.Output, "output", "o", "", "File to write output to"),
+		flagSet.BoolVar(&options.Silent, "silent", false, "Silent mode"),
+		flagSet.BoolVar(&options.Version, "version", false, "Show version"),
+		flagSet.BoolVar(&options.SkipBaseIP, "skip-base", false, "Skip base IPs (ending in .0) in output"),
+		flagSet.BoolVar(&options.SkipBroadcastIP, "skip-broadcast", false, "Skip broadcast IPs (ending in .255) in output"),
+	)
+
+	_ = flagSet.Parse()
 
 	// Read the inputs and configure the logging
 	options.configureOutput()
@@ -281,6 +299,13 @@ func output(wg *sync.WaitGroup, outputchan chan string) {
 		if o == "" {
 			continue
 		}
+		if options.SkipBaseIP && mapcidr.IsBaseIP(o) {
+			continue
+		}
+		if options.SkipBroadcastIP && mapcidr.IsBroadcastIP(o) {
+			continue
+		}
+
 		gologger.Silent().Msgf("%s\n", o)
 		if f != nil {
 			_, _ = f.WriteString(o + "\n")
@@ -298,3 +323,10 @@ func hasStdin() bool {
 	}
 	return true
 }
+
+func createGroup(flagSet *goflags.FlagSet, groupName, description string, flags ...*goflags.FlagData) {
+	flagSet.SetGroup(groupName, description)
+	for _, currentFlag := range flags {
+		currentFlag.Group(groupName)
+	}
+}

go.mod

@@ -4,17 +4,22 @@ go 1.17
 
 require (
 	github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e
+	github.com/projectdiscovery/goflags v0.0.7
 	github.com/projectdiscovery/gologger v1.1.4
 	github.com/projectdiscovery/ipranger v0.0.2
 )
 
 require (
+	github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect
 	github.com/golang/snappy v0.0.3 // indirect
 	github.com/json-iterator/go v1.1.11 // indirect
 	github.com/logrusorgru/aurora v2.0.3+incompatible // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.1 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
 	github.com/projectdiscovery/hmap v0.0.1 // indirect
+	github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe // indirect
 	github.com/syndtr/goleveldb v1.0.0 // indirect
 	github.com/yl2chen/cidranger v1.0.2 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
 )