Add terminator chaos testing and fix issues found. Fixes #1794 Fixes #…

…1369

common/ctrl_msg/messages.go

@@ -55,6 +55,10 @@ const (
 	CreateCircuitRespCircuitId  = 11
 	CreateCircuitRespAddress    = 12
 	CreateCircuitRespTagsHeader = 13
+
+	HeaderResultErrorCode = 10
+
+	ResultErrorRateLimited = 1
 )
 
 func NewCircuitSuccessMsg(sessionId, address string) *channel.Message {

common/handler_common/common.go

@@ -3,6 +3,7 @@ package handler_common
 import (
 	"github.com/michaelquigley/pfxlog"
 	"github.com/openziti/channel/v2"
+	"github.com/openziti/ziti/common/ctrl_msg"
 	"time"
 )
 
@@ -39,3 +40,20 @@ func SendOpResult(request *channel.Message, ch channel.Channel, op string, messa
 		log.WithError(err).Error("failed to send result")
 	}
 }
+
+func SendServerBusy(request *channel.Message, ch channel.Channel, op string) {
+	log := pfxlog.ContextLogger(ch.Label()).WithField("operation", op)
+	log.Errorf("%v error performing %v: (%s)", ch.LogicalName(), op, "server too busy")
+
+	response := channel.NewResult(false, "server too busy")
+	response.ReplyTo(request)
+	response.Headers.PutUint32Header(ctrl_msg.HeaderResultErrorCode, ctrl_msg.ResultErrorRateLimited)
+	if err := response.WithTimeout(5 * time.Second).SendAndWaitForWire(ch); err != nil {
+		log.WithError(err).Error("failed to send result")
+	}
+}
+
+func WasRateLimited(msg *channel.Message) bool {
+	val, found := msg.GetUint32Header(ctrl_msg.HeaderResultErrorCode)
+	return found && val == ctrl_msg.ResultErrorRateLimited
+}

common/inspect/router_message_inspections.go

@@ -0,0 +1,41 @@
+/*
+	Copyright NetFoundry Inc.
+
+	Licensed under the Apache License, Version 2.0 (the "License");
+	you may not use this file except in compliance with the License.
+	You may obtain a copy of the License at
+
+	https://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+	distributed under the License is distributed on an "AS IS" BASIS,
+	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+	See the License for the specific language governing permissions and
+	limitations under the License.
+*/
+
+package inspect
+
+type RouterMessagingState struct {
+	RouterUpdates         []*RouterUpdates         `json:"routerUpdates"`
+	TerminatorValidations []*TerminatorValidations `json:"terminatorValidations"`
+}
+
+type RouterInfo struct {
+	Id   string `json:"id"`
+	Name string `json:"name"`
+}
+
+type RouterUpdates struct {
+	Router         RouterInfo   `json:"router"`
+	Version        uint32       `json:"version"`
+	ChangedRouters []RouterInfo `json:"changedRouters"`
+	SendInProgress bool         `json:"sendInProgress"`
+}
+
+type TerminatorValidations struct {
+	Router          RouterInfo `json:"router"`
+	Terminators     []string   `json:"terminators"`
+	CheckInProgress bool       `json:"checkInProgress"`
+	LastSend        string     `json:"lastSend"`
+}

common/inspect/terminator_inspections.go

@@ -0,0 +1,39 @@
+/*
+	Copyright NetFoundry Inc.
+
+	Licensed under the Apache License, Version 2.0 (the "License");
+	you may not use this file except in compliance with the License.
+	You may obtain a copy of the License at
+
+	https://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+	distributed under the License is distributed on an "AS IS" BASIS,
+	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+	See the License for the specific language governing permissions and
+	limitations under the License.
+*/
+
+package inspect
+
+type SdkTerminatorInspectResult struct {
+	Entries []*SdkTerminatorInspectDetail `json:"entries"`
+	Errors  []string                      `json:"errors"`
+}
+
+type SdkTerminatorInspectDetail struct {
+	Key             string `json:"key"`
+	Id              string `json:"id"`
+	State           string `json:"state"`
+	Token           string `json:"token"`
+	ListenerId      string `json:"listenerId"`
+	Instance        string `json:"instance"`
+	Cost            uint16 `json:"cost"`
+	Precedence      string `json:"precedence"`
+	AssignIds       bool   `json:"assignIds"`
+	V2              bool   `json:"v2"`
+	PostValidate    bool   `json:"postValidate"`
+	OperationActive bool   `json:"establishActive"`
+	CreateTime      string `json:"createTime"`
+	LastAttempt     string `json:"lastAttempt"`
+}

common/pb/edge_ctrl_pb/edge_ctrl.proto

@@ -244,6 +244,7 @@ enum CreateTerminatorResult {
   FailedIdConflict = 1;
   FailedOther = 2;
   FailedBusy = 3;
+  FailedInvalidSession = 4;
 }
 
 message CreateTerminatorV2Response {

common/pb/mgmt_pb/impl.go

@@ -35,3 +35,15 @@ func (request *ValidateRouterLinksResponse) GetContentType() int32 {
 func (request *RouterLinkDetails) GetContentType() int32 {
 	return int32(ContentType_ValidateRouterLinksResultType)
 }
+
+func (request *ValidateRouterSdkTerminatorsRequest) GetContentType() int32 {
+	return int32(ContentType_ValidateRouterSdkTerminatorsRequestType)
+}
+
+func (request *ValidateRouterSdkTerminatorsResponse) GetContentType() int32 {
+	return int32(ContentType_ValidateRouterSdkTerminatorsResponseType)
+}
+
+func (request *RouterSdkTerminatorsDetails) GetContentType() int32 {
+	return int32(ContentType_ValidateRouterSdkTerminatorsResultType)
+}