From ac83d9356742e000223b5c3dd1dc39c7f51353c7 Mon Sep 17 00:00:00 2001 From: Paul Lorenz Date: Fri, 13 Sep 2024 09:17:19 -0400 Subject: [PATCH] Refactor XT. Add concurrency test. Add inspect support. Fixes #2403 --- common/inspect/terminator_inspections.go | 11 ++ controller/network/network.go | 11 ++ controller/xt/costs.go | 35 ++-- controller/xt/failure.go | 110 ------------- controller/xt/xt.go | 17 +- controller/xt_common/cost_visitor.go | 197 ++++++++++++++++++----- controller/xt_common/failure_test.go | 158 ++++++++++++++++++ controller/xt_smartrouting/impl.go | 19 +-- controller/xt_sticky/impl.go | 19 +-- controller/xt_weighted/impl.go | 16 +- ziti/cmd/fabric/inspect.go | 1 + 11 files changed, 370 insertions(+), 224 deletions(-) delete mode 100644 controller/xt/failure.go create mode 100644 controller/xt_common/failure_test.go diff --git a/common/inspect/terminator_inspections.go b/common/inspect/terminator_inspections.go index f12391fde..ac48ff390 100644 --- a/common/inspect/terminator_inspections.go +++ b/common/inspect/terminator_inspections.go @@ -16,6 +16,17 @@ package inspect +type TerminatorCostDetails struct { + Terminators []*TerminatorCostDetail `json:"terminators"` +} + +type TerminatorCostDetail struct { + TerminatorId string `json:"terminatorId"` + CircuitCount uint32 `json:"circuitCount"` + FailureCost uint32 `json:"failureCost"` + CurrentCost uint32 `json:"currentCost"` +} + type SdkTerminatorInspectResult struct { Entries []*SdkTerminatorInspectDetail `json:"entries"` Errors []string `json:"errors"` diff --git a/controller/network/network.go b/controller/network/network.go index 871174e21..3dd73892d 100644 --- a/controller/network/network.go +++ b/controller/network/network.go @@ -1294,6 +1294,17 @@ func (network *Network) Inspect(name string) (*string, error) { } resultStr := string(result) return &resultStr, nil + } else if strings.HasPrefix(lc, "terminator-costs") { + state := inspect.TerminatorCostDetails{} + xt.GlobalCosts().IterCosts(func(terminatorId string, cost xt.Cost) { + state.Terminators = append(state.Terminators, cost.Inspect(terminatorId)) + }) + result, err := json.Marshal(state) + if err != nil { + return nil, fmt.Errorf("failed to marshall terminator cost state to json (%w)", err) + } + resultStr := string(result) + return &resultStr, nil } else { for _, inspectTarget := range network.inspectionTargets.Value() { if handled, val, err := inspectTarget(lc); handled { diff --git a/controller/xt/costs.go b/controller/xt/costs.go index d765628fa..47b5230ed 100644 --- a/controller/xt/costs.go +++ b/controller/xt/costs.go @@ -17,6 +17,7 @@ package xt import ( + "github.com/openziti/ziti/common/inspect" "math" cmap "github.com/orcaman/concurrent-map/v2" @@ -30,7 +31,12 @@ const ( ) var globalCosts = &costs{ - costMap: cmap.New[uint16](), + costMap: cmap.New[Cost](), +} + +type Cost interface { + Get() uint16 + Inspect(terminatorId string) *inspect.TerminatorCostDetail } func GlobalCosts() Costs { @@ -126,32 +132,33 @@ func GetPrecedenceForName(name string) Precedence { } type costs struct { - costMap cmap.ConcurrentMap[string, uint16] + costMap cmap.ConcurrentMap[string, Cost] } func (self *costs) ClearCost(terminatorId string) { self.costMap.Remove(terminatorId) } -func (self *costs) SetDynamicCost(terminatorId string, cost uint16) { - self.costMap.Set(terminatorId, cost) +func (self *costs) SetDynamicCost(terminatorId string, c Cost) { + self.costMap.Set(terminatorId, c) } -func (self *costs) UpdateDynamicCost(terminatorId string, updateF func(uint16) uint16) { - self.costMap.Upsert(terminatorId, 0, func(exist bool, valueInMap uint16, newValue uint16) uint16 { - if !exist { - return updateF(0) - } - - return updateF(valueInMap) - }) +func (self *costs) GetDynamicCost(terminatorId string) uint16 { + if cost, found := self.costMap.Get(terminatorId); found { + return cost.Get() + } + return 0 } -func (self *costs) GetDynamicCost(terminatorId string) uint16 { +func (self *costs) GetCost(terminatorId string) Cost { if cost, found := self.costMap.Get(terminatorId); found { return cost } - return 0 + return nil +} + +func (self *costs) IterCosts(f func(string, Cost)) { + self.costMap.IterCb(f) } // In a list which is sorted by precedence, returns the terminators which have the diff --git a/controller/xt/failure.go b/controller/xt/failure.go deleted file mode 100644 index 8e31a6dc2..000000000 --- a/controller/xt/failure.go +++ /dev/null @@ -1,110 +0,0 @@ -package xt - -import ( - cmap "github.com/orcaman/concurrent-map/v2" - "time" -) - -type failureCosts struct { - costMap cmap.ConcurrentMap[string, uint16] - maxFailureCost uint32 - failureCost uint16 - successCredit uint16 -} - -func NewFailureCosts(maxFailureCost uint16, failureCost uint8, successCredit uint8) FailureCosts { - result := &failureCosts{ - costMap: cmap.New[uint16](), - maxFailureCost: uint32(maxFailureCost), - failureCost: uint16(failureCost), - successCredit: uint16(successCredit), - } - - return result -} - -func (self *failureCosts) CreditOverTime(credit uint8, period time.Duration) *time.Ticker { - ticker := time.NewTicker(period) - go func() { - for range ticker.C { - var keys []string - self.costMap.IterCb(func(key string, _ uint16) { - keys = append(keys, key) - }) - - for _, key := range keys { - actualCredit := self.successWithCredit(key, uint16(credit)) - GlobalCosts().UpdateDynamicCost(key, func(u uint16) uint16 { - if u < actualCredit { - return 0 - } - return u - actualCredit - }) - } - } - }() - return ticker -} - -func (self *failureCosts) Clear(terminatorId string) { - self.costMap.Remove(terminatorId) -} - -func (self *failureCosts) Failure(terminatorId string) uint16 { - var change uint16 - self.costMap.Upsert(terminatorId, 0, func(exist bool, currentCost uint16, newValue uint16) uint16 { - if !exist { - change = self.failureCost - return self.failureCost - } - - nextCost := uint32(currentCost) + uint32(self.failureCost) - if nextCost > self.maxFailureCost { - change = uint16(self.maxFailureCost - uint32(currentCost)) - return uint16(self.maxFailureCost) - } - change = self.failureCost - return uint16(nextCost) - }) - return change -} - -func (self *failureCosts) Success(terminatorId string) uint16 { - return self.successWithCredit(terminatorId, self.successCredit) -} - -func (self *failureCosts) successWithCredit(terminatorId string, credit uint16) uint16 { - val, found := self.costMap.Get(terminatorId) - if !found { - return 0 - } - - if val == 0 { - removed := self.costMap.RemoveCb(terminatorId, func(key string, currentVal uint16, exists bool) bool { - if !exists { - return true - } - - return currentVal == 0 - }) - if removed { - return 0 - } - } - - var change uint16 - self.costMap.Upsert(terminatorId, 0, func(exist bool, currentCost uint16, newValue uint16) uint16 { - if !exist { - change = 0 - return 0 - } - - if currentCost < credit { - change = currentCost - return 0 - } - change = credit - return currentCost - credit - }) - return change -} diff --git a/controller/xt/xt.go b/controller/xt/xt.go index b720eb7e3..730bea678 100644 --- a/controller/xt/xt.go +++ b/controller/xt/xt.go @@ -95,21 +95,10 @@ type EventVisitor interface { VisitCircuitRemoved(event TerminatorEvent) } -type Stats interface { - GetCost() uint32 - GetPrecedence() Precedence -} - type Costs interface { ClearCost(terminatorId string) - SetDynamicCost(terminatorId string, weight uint16) - UpdateDynamicCost(terminatorId string, updateF func(uint16) uint16) + SetDynamicCost(terminatorId string, c Cost) GetDynamicCost(terminatorId string) uint16 -} - -type FailureCosts interface { - Failure(terminatorId string) uint16 - Success(terminatorId string) uint16 - Clear(terminatorId string) - CreditOverTime(credit uint8, period time.Duration) *time.Ticker + GetCost(terminatorId string) Cost + IterCosts(func(terminatorId string, cost Cost)) } diff --git a/controller/xt_common/cost_visitor.go b/controller/xt_common/cost_visitor.go index fd038391c..fa0ce82b7 100644 --- a/controller/xt_common/cost_visitor.go +++ b/controller/xt_common/cost_visitor.go @@ -1,60 +1,181 @@ package xt_common import ( + "github.com/openziti/ziti/common/inspect" "github.com/openziti/ziti/controller/xt" + cmap "github.com/orcaman/concurrent-map/v2" "math" + "sync/atomic" + "time" ) +type TerminatorCosts struct { + CircuitCount uint32 + FailureCost uint32 + CachedCost uint32 +} + +func (self *TerminatorCosts) cache(circuitCost uint32) { + cost := uint64(self.CircuitCount)*uint64(circuitCost) + uint64(self.FailureCost) + if cost > math.MaxUint32 { + cost = math.MaxUint32 + } + atomic.StoreUint32(&self.CachedCost, uint32(cost)) +} + +func (self *TerminatorCosts) Get() uint16 { + val := atomic.LoadUint32(&self.CachedCost) + if val > math.MaxUint16 { + return math.MaxUint16 + } + return uint16(val) +} + +func (self *TerminatorCosts) Inspect(terminatorId string) *inspect.TerminatorCostDetail { + return &inspect.TerminatorCostDetail{ + TerminatorId: terminatorId, + CircuitCount: self.CircuitCount, + FailureCost: self.FailureCost, + CurrentCost: self.CachedCost, + } +} + +func NewCostVisitor(circuitCost, failureCost, successCredit uint16) *CostVisitor { + return &CostVisitor{ + Costs: cmap.New[*TerminatorCosts](), + CircuitCost: uint32(circuitCost), + FailureCost: uint32(failureCost), + SuccessCredit: uint32(successCredit), + } +} + type CostVisitor struct { - FailureCosts xt.FailureCosts - CircuitCost uint16 + Costs cmap.ConcurrentMap[string, *TerminatorCosts] + CircuitCost uint32 + FailureCost uint32 + SuccessCredit uint32 } -func (visitor *CostVisitor) VisitDialFailed(event xt.TerminatorEvent) { - change := visitor.FailureCosts.Failure(event.GetTerminator().GetId()) +func (self *CostVisitor) GetFailureCost(terminatorId string) uint32 { + val, _ := self.Costs.Get(terminatorId) + if val == nil { + return 0 + } + return val.FailureCost +} - if change > 0 { - xt.GlobalCosts().UpdateDynamicCost(event.GetTerminator().GetId(), func(cost uint16) uint16 { - if cost < (math.MaxUint16 - change) { - return cost + change - } - return math.MaxUint16 - }) +func (self *CostVisitor) GetCircuitCount(terminatorId string) uint32 { + val, _ := self.Costs.Get(terminatorId) + if val == nil { + return 0 + } + return val.CircuitCount +} + +func (self *CostVisitor) GetCost(terminatorId string) uint32 { + val, _ := self.Costs.Get(terminatorId) + if val == nil { + return 0 } + return atomic.LoadUint32(&val.CachedCost) } -func (visitor *CostVisitor) VisitDialSucceeded(event xt.TerminatorEvent) { - credit := visitor.FailureCosts.Success(event.GetTerminator().GetId()) - if credit != visitor.CircuitCost { - xt.GlobalCosts().UpdateDynamicCost(event.GetTerminator().GetId(), func(cost uint16) uint16 { - if visitor.CircuitCost > credit { - increase := visitor.CircuitCost - credit - if cost < (math.MaxUint16 - increase) { - // pfxlog.Logger().Infof("%v: dial+ %v -> %v", event.GetTerminator().GetId(), cost, cost+increase) - return cost + increase - } - // pfxlog.Logger().Infof("%v: dial+ %v -> %v", event.GetTerminator().GetId(), cost, math.MaxUint16) - return math.MaxUint16 +func (self *CostVisitor) VisitDialFailed(event xt.TerminatorEvent) { + self.Costs.Upsert(event.GetTerminator().GetId(), nil, func(exist bool, valueInMap *TerminatorCosts, newValue *TerminatorCosts) *TerminatorCosts { + cost := valueInMap + if !exist { + cost = &TerminatorCosts{} + xt.GlobalCosts().SetDynamicCost(event.GetTerminator().GetId(), cost) + } + + if math.MaxUint32-cost.FailureCost > self.FailureCost { + cost.FailureCost += self.FailureCost + } else { + cost.FailureCost = math.MaxUint32 + } + cost.cache(self.CircuitCost) + return cost + }) +} + +func (self *CostVisitor) VisitDialSucceeded(event xt.TerminatorEvent) { + self.Costs.Upsert(event.GetTerminator().GetId(), nil, func(exist bool, valueInMap *TerminatorCosts, newValue *TerminatorCosts) *TerminatorCosts { + cost := valueInMap + if !exist { + cost = &TerminatorCosts{} + xt.GlobalCosts().SetDynamicCost(event.GetTerminator().GetId(), cost) + } + + if cost.FailureCost > self.SuccessCredit { + cost.FailureCost -= self.SuccessCredit + } else { + cost.FailureCost = 0 + } + + if cost.CircuitCount < math.MaxUint32/self.CircuitCost { + cost.CircuitCount++ + } + cost.cache(self.CircuitCost) + return cost + }) +} + +func (self *CostVisitor) VisitCircuitRemoved(event xt.TerminatorEvent) { + self.Costs.Upsert(event.GetTerminator().GetId(), nil, func(exist bool, valueInMap *TerminatorCosts, newValue *TerminatorCosts) *TerminatorCosts { + cost := valueInMap + if !exist { + cost = &TerminatorCosts{} + xt.GlobalCosts().SetDynamicCost(event.GetTerminator().GetId(), cost) + } + + if cost.CircuitCount > 0 { + cost.CircuitCount-- + } + cost.cache(self.CircuitCost) + return cost + }) +} + +func (self *CostVisitor) CreditOverTime(credit uint8, period time.Duration) *time.Ticker { + ticker := time.NewTicker(period) + go func() { + for range ticker.C { + self.CreditAll(credit) + } + }() + return ticker +} + +func (self *CostVisitor) CreditAll(credit uint8) { + var keys []string + self.Costs.IterCb(func(key string, _ *TerminatorCosts) { + keys = append(keys, key) + }) + + for _, key := range keys { + self.Costs.Upsert(key, nil, func(exist bool, valueInMap *TerminatorCosts, newValue *TerminatorCosts) *TerminatorCosts { + cost := valueInMap + if !exist { + cost = &TerminatorCosts{} + xt.GlobalCosts().SetDynamicCost(key, cost) } - decrease := credit - visitor.CircuitCost - if decrease > cost { - // pfxlog.Logger().Infof("%v: dial+ %v -> %v", event.GetTerminator().GetId(), cost, 0) - return 0 + if cost.FailureCost > uint32(credit) { + cost.FailureCost -= uint32(credit) } - // pfxlog.Logger().Infof("%v: dial+ %v -> %v", event.GetTerminator().GetId(), cost, cost-decrease) - return cost - decrease + cost.cache(self.CircuitCost) + return cost }) } } -func (visitor *CostVisitor) VisitCircuitRemoved(event xt.TerminatorEvent) { - xt.GlobalCosts().UpdateDynamicCost(event.GetTerminator().GetId(), func(cost uint16) uint16 { - if cost > visitor.CircuitCost { - // pfxlog.Logger().Infof("%v: sess- %v -> %v", event.GetTerminator().GetId(), cost, cost-1) - return cost - visitor.CircuitCost - } - // pfxlog.Logger().Infof("%v: sess- %v -> %v", event.GetTerminator().GetId(), cost, 0) - return 0 - }) +func (self *CostVisitor) NotifyEvent(event xt.TerminatorEvent) { + event.Accept(self) +} + +func (self *CostVisitor) HandleTerminatorChange(event xt.StrategyChangeEvent) error { + for _, t := range event.GetRemoved() { + self.Costs.Remove(t.GetId()) + } + return nil } diff --git a/controller/xt_common/failure_test.go b/controller/xt_common/failure_test.go new file mode 100644 index 000000000..0af231f2b --- /dev/null +++ b/controller/xt_common/failure_test.go @@ -0,0 +1,158 @@ +package xt_common + +import ( + "fmt" + "github.com/openziti/ziti/controller/xt" + cmap "github.com/orcaman/concurrent-map/v2" + "math/rand" + "sync" + "testing" + "time" +) + +type mockTerminator struct{} + +func (m mockTerminator) GetId() string { + return "test" +} + +func (m mockTerminator) GetPrecedence() xt.Precedence { + panic("implement me") +} + +func (m mockTerminator) GetCost() uint16 { + panic("implement me") +} + +func (m mockTerminator) GetServiceId() string { + panic("implement me") +} + +func (m mockTerminator) GetInstanceId() string { + panic("implement me") +} + +func (m mockTerminator) GetRouterId() string { + panic("implement me") +} + +func (m mockTerminator) GetBinding() string { + panic("implement me") +} + +func (m mockTerminator) GetAddress() string { + panic("implement me") +} + +func (m mockTerminator) GetPeerData() xt.PeerData { + panic("implement me") +} + +func (m mockTerminator) GetCreatedAt() time.Time { + panic("implement me") +} + +func (m mockTerminator) GetHostId() string { + panic("implement me") +} + +func TestFailures(t *testing.T) { + //t.SkipNow() + costVisitor := &CostVisitor{ + Costs: cmap.New[*TerminatorCosts](), + CircuitCost: 2, + FailureCost: 50, + SuccessCredit: 2, + } + + terminator := mockTerminator{} + + var lock sync.Mutex + + workerCount := 10 + + balances := make(chan int32, workerCount) + + for range workerCount { + go func() { + dial := 0 + done := 0 + fail := 0 + + dialPct := 60 + donePct := 35 + + var localBalance int32 + + for i := range 1000000000 { + next := rand.Intn(100) + // successful dial + if next < dialPct { + if localBalance < 4000 { + evt := xt.NewDialSucceeded(terminator) + costVisitor.VisitDialSucceeded(evt) + localBalance++ + dial++ + } + } else if next < dialPct+donePct { + if localBalance > 0 { + evt := xt.NewCircuitRemoved(terminator) + costVisitor.VisitCircuitRemoved(evt) + localBalance-- + done++ + } + } else { + if rand.Intn(10) == 0 { + evt := xt.NewDialFailedEvent(terminator) + costVisitor.VisitDialFailed(evt) + fail++ + } + } + if i%10 == 0 { + costVisitor.CreditAll(2) + } + + if i%1000000 == 0 { + balances <- localBalance + + lock.Lock() + balance := int32(0) + first := false + select { + case balance = <-balances: + first = true + default: + } + + if first { + for range workerCount - 1 { + balance += <-balances + } + + cost := xt.GlobalCosts().GetDynamicCost("test") + failureCost := int(costVisitor.GetFailureCost("test")) + circuitCount := int(costVisitor.GetCircuitCount("test")) + cachedCost := int(costVisitor.GetCost("test")) + expected := 2*int(balance) + failureCost + fmt.Printf("dial: %d, done: %d, fail: %d, balance %d, cost: %d, expected: %d, delta: %d, failureCost: %d, circuitCount: %d, cachedCost: %d\n", + dial, done, fail, balance, cost, expected, expected-int(cost), failureCost, circuitCount, cachedCost) + if int(cost) > expected { + panic("bad state") + } + } + lock.Unlock() + + if dialPct == 65 { + dialPct = 20 + donePct = 79 + } else { + dialPct = 65 + donePct = 30 + } + } + } + }() + } + + time.Sleep(10 * time.Second) +} diff --git a/controller/xt_smartrouting/impl.go b/controller/xt_smartrouting/impl.go index 7ad5d9ed2..40cb68945 100644 --- a/controller/xt_smartrouting/impl.go +++ b/controller/xt_smartrouting/impl.go @@ -19,7 +19,6 @@ package xt_smartrouting import ( "github.com/openziti/ziti/controller/xt" "github.com/openziti/ziti/controller/xt_common" - "math" "time" ) @@ -46,12 +45,9 @@ func (self *factory) GetStrategyName() string { func (self *factory) NewStrategy() xt.Strategy { strategy := strategy{ - CostVisitor: xt_common.CostVisitor{ - FailureCosts: xt.NewFailureCosts(math.MaxUint16/4, 20, 2), - CircuitCost: 2, - }, + CostVisitor: *xt_common.NewCostVisitor(2, 20, 2), } - strategy.CostVisitor.FailureCosts.CreditOverTime(5, time.Minute) + strategy.CreditOverTime(5, time.Minute) return &strategy } @@ -62,14 +58,3 @@ type strategy struct { func (self *strategy) Select(_ xt.CreateCircuitParams, terminators []xt.CostedTerminator) (xt.CostedTerminator, xt.PeerData, error) { return terminators[0], nil, nil } - -func (self *strategy) NotifyEvent(event xt.TerminatorEvent) { - event.Accept(&self.CostVisitor) -} - -func (self *strategy) HandleTerminatorChange(event xt.StrategyChangeEvent) error { - for _, t := range event.GetRemoved() { - self.FailureCosts.Clear(t.GetId()) - } - return nil -} diff --git a/controller/xt_sticky/impl.go b/controller/xt_sticky/impl.go index 8effac2b6..5bc1a6576 100644 --- a/controller/xt_sticky/impl.go +++ b/controller/xt_sticky/impl.go @@ -20,7 +20,6 @@ import ( "github.com/openziti/ziti/common/ctrl_msg" "github.com/openziti/ziti/controller/xt" "github.com/openziti/ziti/controller/xt_common" - "math" "time" ) @@ -47,12 +46,9 @@ func (self *factory) GetStrategyName() string { func (self *factory) NewStrategy() xt.Strategy { strategy := strategy{ - CostVisitor: xt_common.CostVisitor{ - FailureCosts: xt.NewFailureCosts(math.MaxUint16/4, 20, 2), - CircuitCost: 2, - }, + CostVisitor: *xt_common.NewCostVisitor(2, 20, 2), } - strategy.CostVisitor.FailureCosts.CreditOverTime(5, time.Minute) + strategy.CostVisitor.CreditOverTime(5, time.Minute) return &strategy } @@ -86,14 +82,3 @@ func (self *strategy) Select(params xt.CreateCircuitParams, terminators []xt.Cos ctrl_msg.XtStickinessToken: []byte(result.GetId()), }, nil } - -func (self *strategy) NotifyEvent(event xt.TerminatorEvent) { - event.Accept(&self.CostVisitor) -} - -func (self *strategy) HandleTerminatorChange(event xt.StrategyChangeEvent) error { - for _, t := range event.GetRemoved() { - self.FailureCosts.Clear(t.GetId()) - } - return nil -} diff --git a/controller/xt_weighted/impl.go b/controller/xt_weighted/impl.go index b7845124a..e4a5b7f70 100644 --- a/controller/xt_weighted/impl.go +++ b/controller/xt_weighted/impl.go @@ -19,7 +19,6 @@ package xt_weighted import ( "github.com/openziti/ziti/controller/xt" "github.com/openziti/ziti/controller/xt_common" - "math" "math/rand" "time" ) @@ -42,12 +41,9 @@ func (self *factory) GetStrategyName() string { func (self *factory) NewStrategy() xt.Strategy { strategy := &strategy{ - CostVisitor: xt_common.CostVisitor{ - FailureCosts: xt.NewFailureCosts(math.MaxUint16/4, 20, 2), - CircuitCost: 2, - }, + CostVisitor: *xt_common.NewCostVisitor(2, 20, 2), } - strategy.CostVisitor.FailureCosts.CreditOverTime(5, time.Minute) + strategy.CostVisitor.CreditOverTime(5, time.Minute) return strategy } @@ -87,11 +83,3 @@ func (self *strategy) Select(_ xt.CreateCircuitParams, terminators []xt.CostedTe return terminators[0], nil, nil } - -func (self *strategy) NotifyEvent(event xt.TerminatorEvent) { - event.Accept(&self.CostVisitor) -} - -func (self *strategy) HandleTerminatorChange(xt.StrategyChangeEvent) error { - return nil -} diff --git a/ziti/cmd/fabric/inspect.go b/ziti/cmd/fabric/inspect.go index 705dbd2d9..6e2c65b5d 100644 --- a/ziti/cmd/fabric/inspect.go +++ b/ziti/cmd/fabric/inspect.go @@ -35,6 +35,7 @@ func newInspectCmd(p common.OptionsProvider) *cobra.Command { cmd.AddCommand(action.newInspectSubCmd(p, "router-messaging", "gets information about pending router peer updates and terminator validations")) cmd.AddCommand(action.newInspectSubCmd(p, "router-data-model", "gets information about the router data model")) cmd.AddCommand(action.newInspectSubCmd(p, "router-controllers", "gets information about the state of a router's connections to its controllers")) + cmd.AddCommand(action.newInspectSubCmd(p, "terminator-costs", "gets information about terminator dynamic costs")) inspectCircuitsAction := &InspectCircuitsAction{InspectAction: *newInspectAction(p)} cmd.AddCommand(inspectCircuitsAction.newCobraCmd())