Merge pull request #84 from projectdiscovery/dev

v1.0.0 Release

.github/workflows/codeql-analysis.yml

@@ -27,12 +27,12 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
+      uses: github/codeql-action/init@v2
       with:
         languages: ${{ matrix.language }}
 
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
+      uses: github/codeql-action/autobuild@v2
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      uses: github/codeql-action/analyze@v2

.github/workflows/dockerhub-push.yml

@@ -19,19 +19,19 @@ jobs:
         run: |
           echo "::set-output name=tag::$(curl --silent "https://api.github.com/repos/projectdiscovery/mapcidr/releases/latest" | jq -r .tag_name)"
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v2
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
 
       - name: Login to DockerHub
-        uses: docker/login-action@v1 
+        uses: docker/login-action@v2 
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_TOKEN }}
 
       - name: Build and push
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           context: .
           platforms: linux/amd64,linux/arm64,linux/arm

.github/workflows/lint-test.yml

@@ -16,7 +16,7 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v3
       - name: Run golangci-lint
-        uses: golangci/golangci-lint-action@v3.1.0
+        uses: golangci/golangci-lint-action@v3.2.0
         with:
           version: latest
           args: --timeout 5m

.github/workflows/release-binary.yml

@@ -23,7 +23,7 @@ jobs:
         env: 
           GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
         name: "Create release on GitHub"
-        uses: goreleaser/goreleaser-action@v2
+        uses: goreleaser/goreleaser-action@v3
         with: 
           args: "release --rm-dist"
           version: latest

.gitignore

@@ -1,2 +1,3 @@
 
 cmd/mapcidr/mapcidr
+.vscode

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.18.1-alpine AS build-env
+FROM golang:1.18.3-alpine AS build-env
 RUN go install -v github.com/projectdiscovery/mapcidr/cmd/mapcidr@latest
 
 FROM alpine:latest

Makefile

@@ -3,12 +3,17 @@ GOCMD=go
 GOBUILD=$(GOCMD) build
 GOMOD=$(GOCMD) mod
 GOTEST=$(GOCMD) test
-GOGET=$(GOCMD) get
+GOFLAGS := -v 
+LDFLAGS := -s -w
+
+ifneq ($(shell go env GOOS),darwin)
+LDFLAGS := -extldflags "-static"
+endif
 
 all: build
 build:
-		$(GOBUILD) -v -ldflags="-extldflags=-static" -o "mapcidr" cmd/mapcidr/main.go
+	$(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "mapcidr" cmd/mapcidr/main.go
 test: 
-		$(GOTEST) -v ./...
+	$(GOTEST) -v ./...
 tidy:
-		$(GOMOD) tidy
+	$(GOMOD) tidy

README.md

@@ -33,12 +33,16 @@ mapCIDR is developed to ease load distribution for mass scanning operations, it
   <br>
 </h1>
 
- - CIDR expansion support.
- - CIDR slicing support.
- - CIDR/IP aggregation support.
- - CIDR based IP filter support.
- - IP/PORT shuffling support.
- - STD IN/OUT support.
+ - CIDR expansion support (**default**)
+ - CIDR slicing support (`sbh`, `sbc`)
+ - CIDR/IP aggregation support (`a`, `aa`)
+ - CIDR based IP filter support (`cidr`, `ip`)
+ - CIDR/IP sorting support (`s`, `sr`)
+ - CIDR host count support (`c`)
+ - IP/PORT shuffling support (`si`, `sp`)
+ - IPv4/IPv6 Conversation support (`t4`, `t6`)
+ - IPv4/IPv6 Filter support (`f4`, `f6`)
+ - CIDR STD IN/OUT support
 
 # Installation
 
@@ -56,23 +60,35 @@ This will display help for the tool. Here are all the switches it supports.
 
 ```yaml
 INPUT:
-   -cidr string          CIDR to process
-   -l, -list string      File containing list of CIDRs to process
-   -il, -ip-list string  File containing list of IPs to process
+   -cl, -cidr string[]  CIDR/File containing list of CIDRs to process
+   -il, -ip string[]    IP/File containing list of IPs to process
 
 PROCESS:
-   -sbc int                   Slice CIDRs by given CIDR count
-   -sbh int                   Slice CIDRs by given HOST count
-   -agg, -aggregate           Aggregate IPs/CIDRs into the minimum subnet
-   -sip, -shuffle-ip          Shuffle input ip
-   -sp, -shuffle-port string  Shuffle input ip:port
+   -sbc int                Slice CIDRs by given CIDR count
+   -sbh int                Slice CIDRs by given HOST count
+   -a, -aggregate          Aggregate IPs/CIDRs into minimum subnet
+   -aa, -aggregate-approx  Aggregate sparse IPs/CIDRs into minimum approximated subnet
+   -c, -count              Count number of IPs in given CIDR
+   -t4, -to-ipv4           Convert IPs to IPv4 format
+   -t6, -to-ipv6           Convert IPs to IPv6 format
+
+FILTER:
+   -f4, -filter-ipv4  Filter IPv4 IPs from input
+   -f6, -filter-ipv6  Filter IPv6 IPs from input
+   -skip-base         Skip base IPs (ending in .0) in output
+   -skip-broadcast    Skip broadcast IPs (ending in .255) in output
+
+MISCELLANEOUS:
+   -s, -sort                  Sort input IPs/CIDRs in ascending order
+   -sr, -sort-reverse         Sort input IPs/CIDRs in descending order
+   -si, -shuffle-ip           Shuffle Input IPs in random order
+   -sp, -shuffle-port string  Shuffle Input IP:Port in random order
 
 OUTPUT:
+   -verbose            Verbose mode
    -o, -output string  File to write output to
    -silent             Silent mode
-   -version            Show version
-   -skip-base          Skip base IPs (ending in .0) in output
-   -skip-broadcast     Skip broadcast IPs (ending in .255) in output
+   -version            Show version of the project
 ```
 
 # Running mapCIDR
@@ -109,7 +125,7 @@ mapcidr -cidr 173.0.84.0/24
 173.0.84.16
 ```
 
-### CIDR Slicing by CIDR
+### CIDR Slicing by CIDR Count
 
 In order to slice given CIDR or list of CIDR by CIDR count or slice into multiple and equal smaller subnets, use the following command.
 
@@ -131,7 +147,7 @@ mapcidr -cidr 173.0.84.0/24 -sbc 10 -silent
 173.0.84.224/28
 ```
 
-### CIDR slicing by HOST 
+### CIDR slicing by HOST Count
 
 In order to slice given CIDR for equal number of host count in each CIDR, use the following command.
 
@@ -148,26 +164,98 @@ mapcidr -cidr 173.0.84.0/16 -sbh 20000 -silent
 
 Note: it's possible to obtain a perfect split only when the desired amount of slices or hosts per subnet is a powers of two. Otherwise, the tool will attempt to automatically find the best split strategy to obtain the desired outcome. 
 
-### CIDR/IP aggregation
+### CIDR/IP Aggregation
 
 In order to merge multiple CIDR ranges into smaller subnet block, use the following command.
 
 ```console
-mapcidr -l cidrs.txt -silent -aggregate
+$ mapcidr -cl cidrs.txt -aggregate
 ```
 
 In order to list CIDR blocks for given list of IPs, use the following command.
 
 ```console
-mapcidr -l ips.txt -silent -aggregate
+$ mapcidr -il ips.txt -aggregate
 ```
 
-### CIDR based IP filtering
+It's also possible to perform approximated aggregations for sparse ips groups (only version 4). The final interval will contain contiguous ips not belonging to the input:
+
+```console
+$ cat ips.txt 
+
+1.1.1.1
+1.1.1.16
+1.1.1.31
+```
+
+```console
+$ cat ips.txt | mapcidr -aggregate-approx
+
+1.1.1.0/27
+```
+
+### CIDR based IP Filtering
 
 In order to filter IPs from the given list of CIDR ranges, use the following command.
 
 ```console
-mapcidr -ips ip-list.txt -l cirds.txt
+$ mapcidr -il ip-list.txt -cl cirds.txt
+```
+
+### IPS Conversion
+
+**IPv4 | IPv6** addresses can be converted from either the v6 to v4 notation or IPv4-mapped notation into IPv4 addresses using `-t4` and `-t6` to IPv4 and IPv6 respectively.
+
+```console
+$ cat ips.txt 
+
+1.1.1.1
+2.2.2.2
+```
+
+```
+$ mapcidr -il ipv4-list.txt -t6
+
+00:00:00:00:00:ffff:0101:0101
+00:00:00:00:00:ffff:0202:0202
+```
+
+<table>
+<tr>
+<td>
+<h3>Note:</h3>
+
+Not all IPv6 address can be converted to IPv4. You can only convert valid IPv4 represented IPv6 addresses.
+</td>
+</tr>
+</table>
+
+### IPS Filtering
+
+**IPv4 | IPv6** addresses can be filtered from an input list containing both IPv4/IPv6 formatted IPs using `-f4` and `-f6` flag.
+
+
+```console
+$ cat ips.txt 
+
+1.1.1.1
+00:00:00:00:00:ffff:ad00:5400
+```
+
+```console
+$ mapcidr -il ips.txt -f4
+
+1.1.1.1
+```
+
+### CIDR Host Counting
+
+In order to count number of hosts for a given CIDR or list of CIDR, use the following command.
+
+```console
+$ echo 173.0.84.0/16 | mapcidr -count -silent
+
+65536
 ```
 
 # Use mapCIDR as a library

cidr.go

@@ -2,7 +2,6 @@
 package mapcidr
 
 import (
-	"encoding/binary"
 	"fmt"
 	"math"
 	"math/big"
@@ -219,19 +218,6 @@ func currentSubnet(network *net.IPNet, prefixLen int) (*net.IPNet, error) {
 	return &net.IPNet{IP: currentFirst.Mask(mask), Mask: mask}, nil
 }
 
-// func previousSubnet(network *net.IPNet, prefixLen int) (*net.IPNet, bool) {
-// 	startIP := network.IP
-// 	previousIP := make(net.IP, len(startIP))
-// 	copy(previousIP, startIP)
-// 	cMask := net.CIDRMask(prefixLen, 8*len(previousIP))
-// 	previousIP = dec(previousIP)
-// 	previous := &net.IPNet{IP: previousIP.Mask(cMask), Mask: cMask}
-// 	if startIP.Equal(net.IPv4zero) || startIP.Equal(net.IPv6zero) {
-// 		return previous, true
-// 	}
-// 	return previous, false
-// }
-
 // nextSubnet returns the next subnet for an ipnet
 func nextSubnet(network *net.IPNet, prefixLen int) (*net.IPNet, error) {
 	_, currentLast, err := AddressRange(network)
@@ -282,36 +268,30 @@ func IPAddressesAsStream(cidr string) (chan string, error) {
 	if err != nil {
 		return nil, err
 	}
-	return ipAddresses(ipnet), nil
+	return IpAddresses(ipnet), nil
 }
 
 // IPAddressesIPnet returns all IP addresses in an IPNet.
 func IPAddressesIPnet(ipnet *net.IPNet) (ips []string) {
-	for ip := range ipAddresses(ipnet) {
+	for ip := range IpAddresses(ipnet) {
 		ips = append(ips, ip)
 	}
 	return ips
 }
 
-func ipAddresses(ipnet *net.IPNet) (ips chan string) {
+// IpAddresses as stream
+func IpAddresses(ipnet *net.IPNet) (ips chan string) {
 	ips = make(chan string)
 	go func() {
 		defer close(ips)
 
-		// convert IPNet struct mask and address to uint32
-		mask := binary.BigEndian.Uint32(ipnet.Mask)
-		start := binary.BigEndian.Uint32(ipnet.IP)
-
-		// find the final address
-		finish := (start & mask) | (mask ^ 0xffffffff)
-
-		// loop through addresses as uint32
-		for i := start; i <= finish; i++ {
-			// convert back to net.IP
-			ip := make(net.IP, 4) //nolint
-			binary.BigEndian.PutUint32(ip, i)
+		netWithRange := ipNetToRange(*ipnet)
+		for ip := *netWithRange.First; !ip.Equal(*netWithRange.Last); ip = GetNextIP(ip) {
 			ips <- ip.String()
 		}
+
+		// Add the last IP
+		ips <- netWithRange.Last.String()
 	}()
 	return ips
 }