Optimize algorithm

cmd/subcmds/optimizeSG.go

@@ -8,7 +8,7 @@ package subcmds
 import (
 	"github.com/spf13/cobra"
 
-	"github.com/np-guard/vpc-network-config-synthesis/pkg/optimize"
+	sgoptimizer "github.com/np-guard/vpc-network-config-synthesis/pkg/optimize/sg"
 )
 
 const sgNameFlag = "sg-name"
@@ -20,7 +20,7 @@ func NewOptimizeSGCommand(args *inArgs) *cobra.Command {
 		Long:  `OptimizeSG attempts to reduce the number of security group rules in a SG without changing the semantic.`,
 		Args:  cobra.NoArgs,
 		RunE: func(cmd *cobra.Command, _ []string) error {
-			return optimization(cmd, args, optimize.NewSGOptimizer, true)
+			return optimization(cmd, args, sgoptimizer.NewSGOptimizer, true)
 		},
 	}
 

go.mod

@@ -5,7 +5,7 @@ go 1.23.0
 require (
 	github.com/IBM/vpc-go-sdk v0.60.0
 	github.com/np-guard/cloud-resource-collector v0.15.0
-	github.com/np-guard/models v0.5.0
+	github.com/np-guard/models v0.5.1
 	github.com/spf13/cobra v1.8.1
 )
 

go.sum

@@ -134,8 +134,8 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWb
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/np-guard/cloud-resource-collector v0.15.0 h1:jkmxql6D1uBr/qmSOsBzUgeDxlUXSCe7dBKfqfK+QZ4=
 github.com/np-guard/cloud-resource-collector v0.15.0/go.mod h1:klCHnNnuuVcCtGQHA7R1a8fqnvfMCk/5Jdld6V7sN2A=
-github.com/np-guard/models v0.5.0 h1:P37gCg3RD23hZHymFWtthrF+mGIwyHJkWy0wIWIzokQ=
-github.com/np-guard/models v0.5.0/go.mod h1:29M8utxinyUpYaDuIuOyCcMBf7EsMWZcIrRWCjFm0Bw=
+github.com/np-guard/models v0.5.1 h1:qxewCB3cBLkBdcpMk05gKJkV1D7qkbteQdIXbN1juW0=
+github.com/np-guard/models v0.5.1/go.mod h1:29M8utxinyUpYaDuIuOyCcMBf7EsMWZcIrRWCjFm0Bw=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
 github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=

pkg/io/confio/parse_sgs.go

@@ -122,7 +122,7 @@ func translateSGRuleProtocolIcmp(rule *vpcv1.SecurityGroupRuleSecurityGroupRuleP
 	if err != nil {
 		return nil, err
 	}
-	protocol, err := netp.ICMPFromTypeAndCode64(rule.Type, rule.Code)
+	protocol, err := netp.ICMPFromTypeAndCode64WithoutRFCValidation(rule.Type, rule.Code)
 	if err != nil {
 		return nil, err
 	}

pkg/io/tfio/acl.go

@@ -65,7 +65,7 @@ func aclRule(rule *ir.ACLRule, name string) (tf.Block, error) {
 		{Name: "destination", Value: quote(rule.Destination.String())},
 	}
 
-	comment := ""
+	comment := "\n"
 	if rule.Explanation != "" {
 		comment = fmt.Sprintf("# %v", rule.Explanation)
 	}
@@ -109,7 +109,7 @@ func aclCollection(t *ir.ACLCollection, vpc string) (*tf.ConfigFile, error) {
 	var acls = make([]tf.Block, len(sortedACLs))
 	i := 0
 	for _, subnet := range sortedACLs {
-		comment := ""
+		comment := "\n"
 		vpcName := ir.VpcFromScopedResource(subnet)
 		acl := t.ACLs[vpcName][subnet]
 		if len(sortedACLs) > 1 { // not a single nacl

pkg/io/tfio/sg.go

@@ -93,8 +93,8 @@ func sgRule(rule *ir.SGRule, sgName ir.SGName, i int) (tf.Block, error) {
 func sg(sgName, vpcName string) (tf.Block, error) {
 	tfSGName := ir.ChangeScoping(sgName)
 	comment := fmt.Sprintf("\n### SG attached to %s", sgName)
-	if sgName == tfSGName { // its optimization
-		comment = ""
+	if sgName == tfSGName { // optimization mode
+		comment = "\n"
 	}
 	if err := verifyName(tfSGName); err != nil {
 		return tf.Block{}, err

pkg/ir/sg.go

@@ -79,6 +79,11 @@ func (r *SGRule) mustSupersede(other *SGRule) bool {
 	return res
 }
 
+func NewSGRule(direction Direction, remote RemoteType, p netp.Protocol, local *netset.IPBlock, e string) *SGRule {
+	return &SGRule{Direction: direction, Remote: remote, Protocol: p,
+		Local: local, Explanation: e}
+}
+
 func NewSG(sgName SGName) *SG {
 	return &SG{SGName: sgName, InboundRules: []*SGRule{}, OutboundRules: []*SGRule{}, Attached: []ID{}}
 }

pkg/optimize/common.go

@@ -5,9 +5,76 @@ SPDX-License-Identifier: Apache-2.0
 
 package optimize
 
-import "github.com/np-guard/vpc-network-config-synthesis/pkg/ir"
+import (
+	"sort"
+
+	"github.com/np-guard/models/pkg/ds"
+	"github.com/np-guard/models/pkg/netp"
+	"github.com/np-guard/models/pkg/netset"
+
+	"github.com/np-guard/vpc-network-config-synthesis/pkg/ir"
+)
 
 type Optimizer interface {
 	// attempts to reduce number of SG/nACL rules
 	Optimize() (ir.Collection, error)
 }
+
+// each IPBlock is a single CIDR. The CIDRs are disjoint.
+func SortPartitionsByIPAddrs[T any](p []ds.Pair[*netset.IPBlock, T]) []ds.Pair[*netset.IPBlock, T] {
+	cmp := func(i, j int) bool {
+		if p[i].Left.FirstIPAddress() == p[j].Left.FirstIPAddress() {
+			return p[i].Left.LastIPAddress() < p[j].Left.LastIPAddress()
+		}
+		return p[i].Left.FirstIPAddress() < p[j].Left.FirstIPAddress()
+	}
+	sort.Slice(p, cmp)
+	return p
+}
+
+// returns true if this<other
+func LessIPBlock(this, other *netset.IPBlock) bool {
+	if this.FirstIPAddress() == other.FirstIPAddress() {
+		return this.LastIPAddress() < other.LastIPAddress()
+	}
+	return this.FirstIPAddress() < other.FirstIPAddress()
+}
+
+func IcmpsetPartitions(icmpset *netset.ICMPSet) []netp.ICMP {
+	result := make([]netp.ICMP, 0)
+	if icmpset.IsAll() {
+		icmp, _ := netp.ICMPFromTypeAndCode64WithoutRFCValidation(nil, nil)
+		return []netp.ICMP{icmp}
+	}
+
+	for _, cube := range icmpset.Partitions() {
+		for _, typeInterval := range cube.Left.Intervals() {
+			for _, icmpType := range typeInterval.Elements() {
+				if cube.Right.Equal(netset.AllICMPCodes()) {
+					icmp, _ := netp.ICMPFromTypeAndCode64WithoutRFCValidation(&icmpType, nil)
+					result = append(result, icmp)
+					continue
+				}
+				for _, codeInterval := range cube.Right.Intervals() {
+					for _, icmpCode := range codeInterval.Elements() {
+						icmp, _ := netp.ICMPFromTypeAndCode64WithoutRFCValidation(&icmpType, &icmpCode)
+						result = append(result, icmp)
+					}
+				}
+			}
+		}
+	}
+	return result
+}
+
+func IcmpRuleToIcmpSet(icmp netp.ICMP) *netset.ICMPSet {
+	if icmp.TypeCode == nil {
+		return netset.AllICMPSet()
+	}
+	icmpType := int64(icmp.TypeCode.Type)
+	if icmp.TypeCode.Code == nil {
+		return netset.NewICMPSet(icmpType, icmpType, int64(netp.MinICMPCode), int64(netp.MaxICMPCode))
+	}
+	icmpCode := int64(*icmp.TypeCode.Code)
+	return netset.NewICMPSet(icmpType, icmpType, icmpCode, icmpCode)
+}

pkg/optimize/sg/ipCubesToRules.go

@@ -0,0 +1,143 @@
+/*
+Copyright 2023- IBM Inc. All Rights Reserved.
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package sgoptimizer
+
+import (
+	"github.com/np-guard/models/pkg/ds"
+	"github.com/np-guard/models/pkg/interval"
+	"github.com/np-guard/models/pkg/netp"
+	"github.com/np-guard/models/pkg/netset"
+
+	"github.com/np-guard/vpc-network-config-synthesis/pkg/ir"
+	"github.com/np-guard/vpc-network-config-synthesis/pkg/optimize"
+)
+
+// all protocol cubes, represented by a single ipblock that will be decomposed
+// into cidrs. Each cidr will be the remote of a SG rule
+func allProtocolIPCubesIPToRules(cubes *netset.IPBlock, direction ir.Direction) []*ir.SGRule {
+	result := make([]*ir.SGRule, 0)
+	for _, cidr := range cubes.SplitToCidrs() {
+		result = append(result, ir.NewSGRule(direction, cidr, netp.AnyProtocol{}, netset.GetCidrAll(), ""))
+	}
+	return result
+}
+
+// tcpudpIPCubesToRules converts cubes representing tcp or udp protocol rules to SG rules
+func tcpudpIPCubesToRules(cubes []ds.Pair[*netset.IPBlock, *netset.PortSet], allCubes *netset.IPBlock, direction ir.Direction,
+	isTCP bool) []*ir.SGRule {
+	if len(cubes) == 0 {
+		return []*ir.SGRule{}
+	}
+
+	activeRules := make(map[*netset.IPBlock]netp.Protocol) // the key is the first IP
+	result := make([]*ir.SGRule, 0)
+
+	for i := range cubes {
+		// if it is not possible to continue the rule between the cubes, generate all the existing rules
+		if i > 0 && !continuation(cubes[i-1], cubes[i], allCubes) {
+			result = append(result, createActiveRules(activeRules, cubes[i-1].Left.LastIPAddressObject(), direction)...)
+			activeRules = make(map[*netset.IPBlock]netp.Protocol)
+		}
+
+		// if the proctol is not contained in the current cube, we will generate SG rules
+		// calculate active ports = active rules covers these ports
+		activePorts := interval.NewCanonicalSet()
+		for ipb, protocol := range activeRules {
+			if tcpudp, ok := protocol.(netp.TCPUDP); ok {
+				if !tcpudp.DstPorts().ToSet().IsSubset(cubes[i].Right) {
+					result = append(result, createNewRules(protocol, ipb, cubes[i-1].Left.LastIPAddressObject(), direction)...)
+				} else {
+					activePorts.AddInterval(tcpudp.DstPorts())
+				}
+			}
+		}
+
+		// if the cube contains ports that are not contained in the active rules, new rules will be created
+		for _, ports := range cubes[i].Right.Intervals() {
+			if !ports.ToSet().IsSubset(activePorts) {
+				p, _ := netp.NewTCPUDP(isTCP, netp.MinPort, netp.MaxPort, int(ports.Start()), int(ports.End()))
+				activeRules[cubes[i].Left.FirstIPAddressObject()] = p
+			}
+		}
+	}
+	// generate all the existing rules
+	return append(result, createActiveRules(activeRules, cubes[len(cubes)-1].Left.LastIPAddressObject(), direction)...)
+}
+
+// icmpIPCubesToRules converts cubes representing icmp protocol rules to SG rules
+func icmpIPCubesToRules(cubes []ds.Pair[*netset.IPBlock, *netset.ICMPSet], allCubes *netset.IPBlock, direction ir.Direction) []*ir.SGRule {
+	if len(cubes) == 0 {
+		return []*ir.SGRule{}
+	}
+
+	activeRules := make(map[*netset.IPBlock]netp.Protocol) // the key is the first IP
+	result := make([]*ir.SGRule, 0)
+
+	for i := range cubes {
+		// if it is not possible to continue the rule between the cubes, generate all the existing rules
+		if i > 0 && !continuation(cubes[i-1], cubes[i], allCubes) {
+			result = append(result, createActiveRules(activeRules, cubes[i-1].Left.LastIPAddressObject(), direction)...)
+			activeRules = make(map[*netset.IPBlock]netp.Protocol)
+		}
+
+		// if the proctol is not contained in the current cube, we will generate SG rules
+		// calculate activeICMP = active rules covers these icmp values
+		activeICMP := netset.EmptyICMPSet()
+		for ipb, protocol := range activeRules {
+			if icmp, ok := protocol.(netp.ICMP); ok {
+				ruleIcmpSet := optimize.IcmpRuleToIcmpSet(icmp)
+				if !ruleIcmpSet.IsSubset(cubes[i].Right) {
+					result = append(result, createNewRules(protocol, ipb, cubes[i-1].Left.LastIPAddressObject(), direction)...)
+				} else {
+					activeICMP.Union(ruleIcmpSet)
+				}
+			}
+		}
+
+		// if the cube contains icmp values that are not contained in the active rules, new rules will be created
+		for _, p := range optimize.IcmpsetPartitions(cubes[i].Right) {
+			if !optimize.IcmpRuleToIcmpSet(p).IsSubset(activeICMP) {
+				activeRules[cubes[i].Left.FirstIPAddressObject()] = p
+			}
+		}
+	}
+
+	// generate all the existing rules
+	return append(result, createActiveRules(activeRules, cubes[len(cubes)-1].Left.LastIPAddressObject(), direction)...)
+}
+
+// continuation returns true if the rules can be continued between the two cubes
+func continuation[T ds.Set[T]](prevPair, currPair ds.Pair[*netset.IPBlock, T], allProtocolCubes *netset.IPBlock) bool {
+	prevIPBlock := prevPair.Left
+	currIPBlock := currPair.Left
+	touching, _ := prevIPBlock.TouchingIPRanges(currIPBlock)
+	if touching {
+		return true
+	}
+	startH, _ := prevIPBlock.NextIP()
+	endH, _ := currIPBlock.PreviousIP()
+	hole, _ := netset.IPBlockFromIPRange(startH, endH)
+	return hole.IsSubset(allProtocolCubes)
+}
+
+// creates sgRules from SG active rules
+func createActiveRules(activeRules map[*netset.IPBlock]netp.Protocol, endIP *netset.IPBlock, direction ir.Direction) []*ir.SGRule {
+	res := make([]*ir.SGRule, 0)
+	for ipb, protocol := range activeRules {
+		res = append(res, createNewRules(protocol, ipb, endIP, direction)...)
+	}
+	return res
+}
+
+// createNewRules breaks the startIP-endIP ip range into cidrs and creates SG rules
+func createNewRules(protocol netp.Protocol, startIP, endIP *netset.IPBlock, direction ir.Direction) []*ir.SGRule {
+	res := make([]*ir.SGRule, 0)
+	ipRange, _ := netset.IPBlockFromIPRange(startIP, endIP)
+	for _, cidr := range ipRange.SplitToCidrs() {
+		res = append(res, ir.NewSGRule(direction, cidr, protocol, netset.GetCidrAll(), ""))
+	}
+	return res
+}