From fab81e4cd46e584c0ab87188ce9182926085ba53 Mon Sep 17 00:00:00 2001
From: Paul Lorenz <paul.lorenz@netfoundry.io>
Date: Fri, 16 Feb 2024 17:50:13 -0500
Subject: [PATCH] Add terminator chaos testing and fix issues found. Fixes
 #1794 Fixes #1369

---
 CHANGELOG.md                                  |  41 +
 common/ctrl_msg/messages.go                   |   4 +
 common/handler_common/common.go               |  18 +
 ...rcuit_detail.go => circuit_inspections.go} |   0
 ...inspect_result.go => links_inspections.go} |   0
 common/inspect/router_message_inspections.go  |  41 +
 common/inspect/terminator_inspections.go      |  39 +
 common/pb/cmd_pb/cmd.pb.go                    |   2 +-
 common/pb/ctrl_pb/ctrl.pb.go                  |  69 +-
 common/pb/ctrl_pb/ctrl.proto                  |   2 +
 common/pb/edge_cmd_pb/edge_cmd.pb.go          |   2 +-
 common/pb/edge_ctrl_pb/edge_ctrl.pb.go        |  33 +-
 common/pb/edge_ctrl_pb/edge_ctrl.proto        |   1 +
 common/pb/edge_mgmt_pb/edge_mgmt.pb.go        |   2 +-
 common/pb/mgmt_pb/impl.go                     |  12 +
 common/pb/mgmt_pb/mgmt.pb.go                  | 726 ++++++++++---
 common/pb/mgmt_pb/mgmt.proto                  |  32 +
 controller/api_impl/circuit_api_model.go      |  17 +-
 controller/change/util.go                     |  30 +
 controller/command/command.go                 |   5 +
 controller/command/rate_limiter.go            | 337 +++++-
 controller/command/rate_limiter_test.go       |  87 +-
 controller/config/config.go                   |  57 +-
 controller/db/api_session_store.go            |  53 +-
 controller/db/stores.go                       |   9 +-
 controller/db/testing.go                      |   3 +-
 controller/events/metrics_mappers.go          |   7 +-
 controller/handler_ctrl/base.go               |   9 +-
 controller/handler_ctrl/bind.go               |   5 +
 controller/handler_ctrl/remove_terminators.go |   5 +-
 .../handler_edge_ctrl/create_terminator_v2.go |  15 +-
 controller/handler_edge_ctrl/errors.go        |  16 +
 controller/handler_edge_ctrl/errors_test.go   |  29 +
 controller/handler_mgmt/bind.go               |   6 +
 .../validate_router_sdk_terminators.go        |  88 ++
 .../internal/routes/authenticate_router.go    |  26 +-
 controller/internal/routes/identity_router.go |  20 +-
 controller/internal/routes/service_router.go  |  21 +-
 controller/model/identity_manager.go          |   4 +-
 controller/network/network.go                 |  71 +-
 controller/network/router.go                  | 100 ++
 controller/network/router_messaging.go        | 362 ++++++-
 controller/raft/member.go                     |  12 +-
 controller/raft/raft.go                       |   4 +
 go.mod                                        |  22 +-
 go.sum                                        |  42 +-
 router/config.go                              |  59 ++
 router/env/env.go                             |   2 +
 router/fabric/manager.go                      |   7 +-
 router/handler_ctrl/bind.go                   |  38 +-
 router/handler_ctrl/inspect.go                |  27 +-
 .../handler_ctrl/validate_terminators_v2.go   |  84 +-
 router/router.go                              |   7 +
 router/router_test.go                         |   3 +
 router/xgress/request.go                      |   4 +-
 router/xgress/xgress.go                       |   4 +
 router/xgress_edge/dialer.go                  |  12 +-
 router/xgress_edge/fabric.go                  | 202 ++--
 router/xgress_edge/hosted.go                  | 959 ++++++++++++++----
 router/xgress_edge/listener.go                | 180 ++--
 router/xgress_edge/terminator_heap.go         |  45 -
 version                                       |   2 +-
 ziti/cmd/cmd.go                               |   2 +-
 ziti/cmd/database/add_debug_admin.go          |   4 +-
 ziti/cmd/edge/root.go                         |  24 +-
 ziti/cmd/edge/validate_service_hosting.go     | 131 +++
 ziti/cmd/fabric/inspect.go                    |   7 +-
 ziti/cmd/fabric/root.go                       |   1 +
 .../fabric/validate_router_sdk_terminators.go | 158 +++
 ziti/cmd/fabric/validate_terminators.go       |  10 +-
 zititest/go.mod                               |  24 +-
 zititest/go.sum                               |  46 +-
 .../db-sdk-hosting-test/configs/ctrl.yml.tmpl | 205 ++++
 .../configs/router.yml.tmpl                   |  70 ++
 zititest/models/db-sdk-hosting-test/main.go   | 324 ++++++
 zititest/models/links-test/main.go            |   1 -
 .../sdk-hosting-test/configs/ctrl.yml.tmpl    |   4 +
 zititest/models/sdk-hosting-test/main.go      | 444 ++++----
 .../models/sdk-hosting-test/validation.go     | 263 +++++
 .../zitilab/actions/edge/init_identities.go   |   2 +-
 zititest/zitilab/cli/cli.go                   |  41 +
 zititest/zitilab/component_common.go          |   3 +-
 zititest/zitilab/component_ziti_tunnel.go     |  19 +-
 zititest/zitilab/component_zrok_looptest.go   |   2 +-
 zititest/zitilab/models/db_builder.go         |   3 +-
 85 files changed, 4825 insertions(+), 1084 deletions(-)
 rename common/inspect/{circuit_detail.go => circuit_inspections.go} (100%)
 rename common/inspect/{links_inspect_result.go => links_inspections.go} (100%)
 create mode 100644 common/inspect/router_message_inspections.go
 create mode 100644 common/inspect/terminator_inspections.go
 create mode 100644 controller/change/util.go
 create mode 100644 controller/handler_edge_ctrl/errors_test.go
 create mode 100644 controller/handler_mgmt/validate_router_sdk_terminators.go
 delete mode 100644 router/xgress_edge/terminator_heap.go
 create mode 100644 ziti/cmd/edge/validate_service_hosting.go
 create mode 100644 ziti/cmd/fabric/validate_router_sdk_terminators.go
 create mode 100644 zititest/models/db-sdk-hosting-test/configs/ctrl.yml.tmpl
 create mode 100644 zititest/models/db-sdk-hosting-test/configs/router.yml.tmpl
 create mode 100644 zititest/models/db-sdk-hosting-test/main.go
 create mode 100644 zititest/models/sdk-hosting-test/validation.go

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 358d48d86..9080be82f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,44 @@
+# Release 0.33.0
+
+# What's New
+
+* SDK Terminator stability improvements
+* Minor feature updates and bug fixes
+
+## SDK Terminator stability improvements
+
+This release was focused on creating a chaos test for SDK terminators, running it and fixing any issues found.
+The test repeatedly and randomly restarts the controller, routers and tunnelers then verifies that terminators
+end up in the correct state. 
+
+The following tools were also used/added to aid in diagnosing and fixing issues:
+
+* `ziti fabric validate router-sdk-terminators` 
+    * Compares the controller state with the router state
+* `ziti fabric validate terminators`
+    * Checks each selected terminator to ensure it's still valid on the router and/or sdk
+* `ziti fabric inspect sdk-terminators`
+    * Allows inspecting each routers terminator state
+* `ziti fabric inspect router-messaging`
+    * Allows inspecting what the controller has queued for router state sync and terminator validations
+* `ziti edge validate service-hosting`
+    * Shows how many terminators each identity which can host a service has
+
+Several changes were made to the terminator code to ensure that terminators are properly created and cleaned up.
+The routers now use an adaptive rate limiter to control how fast they send terminator related requests to the
+controller. For this to work properly, the rate limiting on the controller must be enabled, so it can report
+back to the routers when it's got too much work.
+
+## Component Updates and Bug Fixes
+
+* github.com/openziti/edge-api: [v0.26.10 -> v0.26.12](https://github.com/openziti/edge-api/compare/v0.26.10...v0.26.12)
+* github.com/openziti/ziti: [v0.32.2 -> v0.33.0](https://github.com/openziti/ziti/compare/v0.32.2...v0.33.0)
+    * [Issue #1794](https://github.com/openziti/ziti/issues/1794) - Add SDK terminator chaos test and fix any bugs found as part of chaos testing
+    * [Issue #1369](https://github.com/openziti/ziti/issues/1369) - Allow filtering by policy type when listing identities for service or services for identity
+    * [Issue #1204](https://github.com/openziti/ziti/issues/1204) - ziti cli identity tags related flags misbehaving
+    * [Issue #987](https://github.com/openziti/ziti/issues/987) - "ziti create config router edge" doesn't know about --tunnelerMode proxy
+    * [Issue #652](https://github.com/openziti/ziti/issues/652) - Update CLI script M1 Support when github actions allows
+
 # Release 0.32.2
 
 ## What's New
diff --git a/common/ctrl_msg/messages.go b/common/ctrl_msg/messages.go
index 2b7191c44..96a8524ab 100644
--- a/common/ctrl_msg/messages.go
+++ b/common/ctrl_msg/messages.go
@@ -55,6 +55,10 @@ const (
 	CreateCircuitRespCircuitId  = 11
 	CreateCircuitRespAddress    = 12
 	CreateCircuitRespTagsHeader = 13
+
+	HeaderResultErrorCode = 10
+
+	ResultErrorRateLimited = 1
 )
 
 func NewCircuitSuccessMsg(sessionId, address string) *channel.Message {
diff --git a/common/handler_common/common.go b/common/handler_common/common.go
index 2993c5f6d..57db44179 100644
--- a/common/handler_common/common.go
+++ b/common/handler_common/common.go
@@ -3,6 +3,7 @@ package handler_common
 import (
 	"github.com/michaelquigley/pfxlog"
 	"github.com/openziti/channel/v2"
+	"github.com/openziti/ziti/common/ctrl_msg"
 	"time"
 )
 
@@ -39,3 +40,20 @@ func SendOpResult(request *channel.Message, ch channel.Channel, op string, messa
 		log.WithError(err).Error("failed to send result")
 	}
 }
+
+func SendServerBusy(request *channel.Message, ch channel.Channel, op string) {
+	log := pfxlog.ContextLogger(ch.Label()).WithField("operation", op)
+	log.Errorf("%v error performing %v: (%s)", ch.LogicalName(), op, "server too busy")
+
+	response := channel.NewResult(false, "server too busy")
+	response.ReplyTo(request)
+	response.Headers.PutUint32Header(ctrl_msg.HeaderResultErrorCode, ctrl_msg.ResultErrorRateLimited)
+	if err := response.WithTimeout(5 * time.Second).SendAndWaitForWire(ch); err != nil {
+		log.WithError(err).Error("failed to send result")
+	}
+}
+
+func WasRateLimited(msg *channel.Message) bool {
+	val, found := msg.GetUint32Header(ctrl_msg.HeaderResultErrorCode)
+	return found && val == ctrl_msg.ResultErrorRateLimited
+}
diff --git a/common/inspect/circuit_detail.go b/common/inspect/circuit_inspections.go
similarity index 100%
rename from common/inspect/circuit_detail.go
rename to common/inspect/circuit_inspections.go
diff --git a/common/inspect/links_inspect_result.go b/common/inspect/links_inspections.go
similarity index 100%
rename from common/inspect/links_inspect_result.go
rename to common/inspect/links_inspections.go
diff --git a/common/inspect/router_message_inspections.go b/common/inspect/router_message_inspections.go
new file mode 100644
index 000000000..1f069d115
--- /dev/null
+++ b/common/inspect/router_message_inspections.go
@@ -0,0 +1,41 @@
+/*
+	Copyright NetFoundry Inc.
+
+	Licensed under the Apache License, Version 2.0 (the "License");
+	you may not use this file except in compliance with the License.
+	You may obtain a copy of the License at
+
+	https://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+	distributed under the License is distributed on an "AS IS" BASIS,
+	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+	See the License for the specific language governing permissions and
+	limitations under the License.
+*/
+
+package inspect
+
+type RouterMessagingState struct {
+	RouterUpdates         []*RouterUpdates         `json:"routerUpdates"`
+	TerminatorValidations []*TerminatorValidations `json:"terminatorValidations"`
+}
+
+type RouterInfo struct {
+	Id   string `json:"id"`
+	Name string `json:"name"`
+}
+
+type RouterUpdates struct {
+	Router         RouterInfo   `json:"router"`
+	Version        uint32       `json:"version"`
+	ChangedRouters []RouterInfo `json:"changedRouters"`
+	SendInProgress bool         `json:"sendInProgress"`
+}
+
+type TerminatorValidations struct {
+	Router          RouterInfo `json:"router"`
+	Terminators     []string   `json:"terminators"`
+	CheckInProgress bool       `json:"checkInProgress"`
+	LastSend        string     `json:"lastSend"`
+}
diff --git a/common/inspect/terminator_inspections.go b/common/inspect/terminator_inspections.go
new file mode 100644
index 000000000..f12391fde
--- /dev/null
+++ b/common/inspect/terminator_inspections.go
@@ -0,0 +1,39 @@
+/*
+	Copyright NetFoundry Inc.
+
+	Licensed under the Apache License, Version 2.0 (the "License");
+	you may not use this file except in compliance with the License.
+	You may obtain a copy of the License at
+
+	https://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+	distributed under the License is distributed on an "AS IS" BASIS,
+	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+	See the License for the specific language governing permissions and
+	limitations under the License.
+*/
+
+package inspect
+
+type SdkTerminatorInspectResult struct {
+	Entries []*SdkTerminatorInspectDetail `json:"entries"`
+	Errors  []string                      `json:"errors"`
+}
+
+type SdkTerminatorInspectDetail struct {
+	Key             string `json:"key"`
+	Id              string `json:"id"`
+	State           string `json:"state"`
+	Token           string `json:"token"`
+	ListenerId      string `json:"listenerId"`
+	Instance        string `json:"instance"`
+	Cost            uint16 `json:"cost"`
+	Precedence      string `json:"precedence"`
+	AssignIds       bool   `json:"assignIds"`
+	V2              bool   `json:"v2"`
+	SupportsInspect bool   `json:"supportsInspect"`
+	OperationActive bool   `json:"establishActive"`
+	CreateTime      string `json:"createTime"`
+	LastAttempt     string `json:"lastAttempt"`
+}
diff --git a/common/pb/cmd_pb/cmd.pb.go b/common/pb/cmd_pb/cmd.pb.go
index fb9961620..8f17c8487 100644
--- a/common/pb/cmd_pb/cmd.pb.go
+++ b/common/pb/cmd_pb/cmd.pb.go
@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.31.0
-// 	protoc        v4.23.4
+// 	protoc        v3.21.12
 // source: cmd.proto
 
 package cmd_pb
diff --git a/common/pb/ctrl_pb/ctrl.pb.go b/common/pb/ctrl_pb/ctrl.pb.go
index 7b258309c..df1085a8a 100644
--- a/common/pb/ctrl_pb/ctrl.pb.go
+++ b/common/pb/ctrl_pb/ctrl.pb.go
@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.31.0
-// 	protoc        v4.23.4
+// 	protoc        v3.21.12
 // source: ctrl.proto
 
 package ctrl_pb
@@ -923,6 +923,7 @@ type Terminator struct {
 	Id      string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
 	Binding string `protobuf:"bytes,2,opt,name=binding,proto3" json:"binding,omitempty"`
 	Address string `protobuf:"bytes,3,opt,name=address,proto3" json:"address,omitempty"`
+	Marker  uint64 `protobuf:"varint,4,opt,name=marker,proto3" json:"marker,omitempty"`
 }
 
 func (x *Terminator) Reset() {
@@ -978,6 +979,13 @@ func (x *Terminator) GetAddress() string {
 	return ""
 }
 
+func (x *Terminator) GetMarker() uint64 {
+	if x != nil {
+		return x.Marker
+	}
+	return 0
+}
+
 type ValidateTerminatorsRequest struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
@@ -1088,6 +1096,7 @@ type RouterTerminatorState struct {
 	Valid  bool                    `protobuf:"varint,1,opt,name=valid,proto3" json:"valid,omitempty"`
 	Reason TerminatorInvalidReason `protobuf:"varint,2,opt,name=reason,proto3,enum=ziti.ctrl.pb.TerminatorInvalidReason" json:"reason,omitempty"`
 	Detail string                  `protobuf:"bytes,3,opt,name=detail,proto3" json:"detail,omitempty"` // inspect info if valid
+	Marker uint64                  `protobuf:"varint,4,opt,name=marker,proto3" json:"marker,omitempty"`
 }
 
 func (x *RouterTerminatorState) Reset() {
@@ -1143,6 +1152,13 @@ func (x *RouterTerminatorState) GetDetail() string {
 	return ""
 }
 
+func (x *RouterTerminatorState) GetMarker() uint64 {
+	if x != nil {
+		return x.Marker
+	}
+	return 0
+}
+
 type ValidateTerminatorsV2Response struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
@@ -2647,34 +2663,37 @@ var file_ctrl_proto_rawDesc = []byte{
 	0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65,
 	0x73, 0x74, 0x12, 0x24, 0x0a, 0x0d, 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72,
 	0x49, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x74, 0x65, 0x72, 0x6d, 0x69,
-	0x6e, 0x61, 0x74, 0x6f, 0x72, 0x49, 0x64, 0x73, 0x22, 0x50, 0x0a, 0x0a, 0x54, 0x65, 0x72, 0x6d,
+	0x6e, 0x61, 0x74, 0x6f, 0x72, 0x49, 0x64, 0x73, 0x22, 0x68, 0x0a, 0x0a, 0x54, 0x65, 0x72, 0x6d,
 	0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01,
 	0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e,
 	0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67,
 	0x12, 0x18, 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28,
-	0x09, 0x52, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x22, 0x58, 0x0a, 0x1a, 0x56, 0x61,
-	0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72,
-	0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3a, 0x0a, 0x0b, 0x74, 0x65, 0x72, 0x6d,
-	0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e,
-	0x7a, 0x69, 0x74, 0x69, 0x2e, 0x63, 0x74, 0x72, 0x6c, 0x2e, 0x70, 0x62, 0x2e, 0x54, 0x65, 0x72,
-	0x6d, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, 0x52, 0x0b, 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61,
-	0x74, 0x6f, 0x72, 0x73, 0x22, 0x7a, 0x0a, 0x1c, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65,
-	0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, 0x73, 0x56, 0x32, 0x52, 0x65, 0x71,
-	0x75, 0x65, 0x73, 0x74, 0x12, 0x3a, 0x0a, 0x0b, 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74,
-	0x6f, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x7a, 0x69, 0x74, 0x69,
-	0x2e, 0x63, 0x74, 0x72, 0x6c, 0x2e, 0x70, 0x62, 0x2e, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61,
-	0x74, 0x6f, 0x72, 0x52, 0x0b, 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, 0x73,
-	0x12, 0x1e, 0x0a, 0x0a, 0x66, 0x69, 0x78, 0x49, 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x18, 0x02,
-	0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x66, 0x69, 0x78, 0x49, 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64,
-	0x22, 0x84, 0x01, 0x0a, 0x15, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x72, 0x54, 0x65, 0x72, 0x6d, 0x69,
-	0x6e, 0x61, 0x74, 0x6f, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61,
-	0x6c, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x69, 0x64,
-	0x12, 0x3d, 0x0a, 0x06, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e,
-	0x32, 0x25, 0x2e, 0x7a, 0x69, 0x74, 0x69, 0x2e, 0x63, 0x74, 0x72, 0x6c, 0x2e, 0x70, 0x62, 0x2e,
-	0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, 0x49, 0x6e, 0x76, 0x61, 0x6c, 0x69,
-	0x64, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x52, 0x06, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x12,
-	0x16, 0x0a, 0x06, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52,
-	0x06, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x22, 0xd0, 0x01, 0x0a, 0x1d, 0x56, 0x61, 0x6c, 0x69,
+	0x09, 0x52, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x6d, 0x61,
+	0x72, 0x6b, 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x6d, 0x61, 0x72, 0x6b,
+	0x65, 0x72, 0x22, 0x58, 0x0a, 0x1a, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x65,
+	0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
+	0x12, 0x3a, 0x0a, 0x0b, 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, 0x73, 0x18,
+	0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x7a, 0x69, 0x74, 0x69, 0x2e, 0x63, 0x74, 0x72,
+	0x6c, 0x2e, 0x70, 0x62, 0x2e, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, 0x52,
+	0x0b, 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, 0x73, 0x22, 0x7a, 0x0a, 0x1c,
+	0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74,
+	0x6f, 0x72, 0x73, 0x56, 0x32, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3a, 0x0a, 0x0b,
+	0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28,
+	0x0b, 0x32, 0x18, 0x2e, 0x7a, 0x69, 0x74, 0x69, 0x2e, 0x63, 0x74, 0x72, 0x6c, 0x2e, 0x70, 0x62,
+	0x2e, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, 0x52, 0x0b, 0x74, 0x65, 0x72,
+	0x6d, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x66, 0x69, 0x78, 0x49,
+	0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x66, 0x69,
+	0x78, 0x49, 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x22, 0x9c, 0x01, 0x0a, 0x15, 0x52, 0x6f, 0x75,
+	0x74, 0x65, 0x72, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, 0x53, 0x74, 0x61,
+	0x74, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28,
+	0x08, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x12, 0x3d, 0x0a, 0x06, 0x72, 0x65, 0x61, 0x73,
+	0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x25, 0x2e, 0x7a, 0x69, 0x74, 0x69, 0x2e,
+	0x63, 0x74, 0x72, 0x6c, 0x2e, 0x70, 0x62, 0x2e, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74,
+	0x6f, 0x72, 0x49, 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x52,
+	0x06, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x65, 0x74, 0x61, 0x69,
+	0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x12,
+	0x16, 0x0a, 0x06, 0x6d, 0x61, 0x72, 0x6b, 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52,
+	0x06, 0x6d, 0x61, 0x72, 0x6b, 0x65, 0x72, 0x22, 0xd0, 0x01, 0x0a, 0x1d, 0x56, 0x61, 0x6c, 0x69,
 	0x64, 0x61, 0x74, 0x65, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, 0x73, 0x56,
 	0x32, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4f, 0x0a, 0x06, 0x73, 0x74, 0x61,
 	0x74, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x7a, 0x69, 0x74, 0x69,
diff --git a/common/pb/ctrl_pb/ctrl.proto b/common/pb/ctrl_pb/ctrl.proto
index 3d1d1d954..eb64a1be2 100644
--- a/common/pb/ctrl_pb/ctrl.proto
+++ b/common/pb/ctrl_pb/ctrl.proto
@@ -109,6 +109,7 @@ message Terminator {
   string id = 1;
   string binding = 2;
   string address = 3;
+  uint64 marker = 4;
 }
 
 message ValidateTerminatorsRequest {
@@ -130,6 +131,7 @@ message RouterTerminatorState {
   bool valid = 1;
   TerminatorInvalidReason reason = 2;
   string detail = 3; // inspect info if valid
+  uint64 marker = 4;
 }
 
 message ValidateTerminatorsV2Response {
diff --git a/common/pb/edge_cmd_pb/edge_cmd.pb.go b/common/pb/edge_cmd_pb/edge_cmd.pb.go
index 3918f58a1..dfa6c0220 100644
--- a/common/pb/edge_cmd_pb/edge_cmd.pb.go
+++ b/common/pb/edge_cmd_pb/edge_cmd.pb.go
@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.31.0
-// 	protoc        v4.23.4
+// 	protoc        v3.21.12
 // source: edge_cmd.proto
 
 package edge_cmd_pb
diff --git a/common/pb/edge_ctrl_pb/edge_ctrl.pb.go b/common/pb/edge_ctrl_pb/edge_ctrl.pb.go
index a50e159cc..b47cbaff0 100644
--- a/common/pb/edge_ctrl_pb/edge_ctrl.pb.go
+++ b/common/pb/edge_ctrl_pb/edge_ctrl.pb.go
@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.31.0
-// 	protoc        v4.23.4
+// 	protoc        v3.21.12
 // source: edge_ctrl.proto
 
 package edge_ctrl_pb
@@ -384,10 +384,11 @@ func (TerminatorPrecedence) EnumDescriptor() ([]byte, []int) {
 type CreateTerminatorResult int32
 
 const (
-	CreateTerminatorResult_Success          CreateTerminatorResult = 0
-	CreateTerminatorResult_FailedIdConflict CreateTerminatorResult = 1
-	CreateTerminatorResult_FailedOther      CreateTerminatorResult = 2
-	CreateTerminatorResult_FailedBusy       CreateTerminatorResult = 3
+	CreateTerminatorResult_Success              CreateTerminatorResult = 0
+	CreateTerminatorResult_FailedIdConflict     CreateTerminatorResult = 1
+	CreateTerminatorResult_FailedOther          CreateTerminatorResult = 2
+	CreateTerminatorResult_FailedBusy           CreateTerminatorResult = 3
+	CreateTerminatorResult_FailedInvalidSession CreateTerminatorResult = 4
 )
 
 // Enum value maps for CreateTerminatorResult.
@@ -397,12 +398,14 @@ var (
 		1: "FailedIdConflict",
 		2: "FailedOther",
 		3: "FailedBusy",
+		4: "FailedInvalidSession",
 	}
 	CreateTerminatorResult_value = map[string]int32{
-		"Success":          0,
-		"FailedIdConflict": 1,
-		"FailedOther":      2,
-		"FailedBusy":       3,
+		"Success":              0,
+		"FailedIdConflict":     1,
+		"FailedOther":          2,
+		"FailedBusy":           3,
+		"FailedInvalidSession": 4,
 	}
 )
 
@@ -4113,16 +4116,18 @@ var file_edge_ctrl_proto_rawDesc = []byte{
 	0x0a, 0x14, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, 0x50, 0x72, 0x65, 0x63,
 	0x65, 0x64, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c,
 	0x74, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x52, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x10,
-	0x01, 0x12, 0x0a, 0x0a, 0x06, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x10, 0x02, 0x2a, 0x5c, 0x0a,
+	0x01, 0x12, 0x0a, 0x0a, 0x06, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x10, 0x02, 0x2a, 0x76, 0x0a,
 	0x16, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x6f,
 	0x72, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x75, 0x63, 0x63, 0x65,
 	0x73, 0x73, 0x10, 0x00, 0x12, 0x14, 0x0a, 0x10, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x49, 0x64,
 	0x43, 0x6f, 0x6e, 0x66, 0x6c, 0x69, 0x63, 0x74, 0x10, 0x01, 0x12, 0x0f, 0x0a, 0x0b, 0x46, 0x61,
 	0x69, 0x6c, 0x65, 0x64, 0x4f, 0x74, 0x68, 0x65, 0x72, 0x10, 0x02, 0x12, 0x0e, 0x0a, 0x0a, 0x46,
-	0x61, 0x69, 0x6c, 0x65, 0x64, 0x42, 0x75, 0x73, 0x79, 0x10, 0x03, 0x42, 0x2a, 0x5a, 0x28, 0x67,
-	0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x7a, 0x69,
-	0x74, 0x69, 0x2f, 0x65, 0x64, 0x67, 0x65, 0x2f, 0x70, 0x62, 0x2f, 0x65, 0x64, 0x67, 0x65, 0x5f,
-	0x63, 0x74, 0x72, 0x6c, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x61, 0x69, 0x6c, 0x65, 0x64, 0x42, 0x75, 0x73, 0x79, 0x10, 0x03, 0x12, 0x18, 0x0a, 0x14, 0x46,
+	0x61, 0x69, 0x6c, 0x65, 0x64, 0x49, 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x53, 0x65, 0x73, 0x73,
+	0x69, 0x6f, 0x6e, 0x10, 0x04, 0x42, 0x2a, 0x5a, 0x28, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e,
+	0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x7a, 0x69, 0x74, 0x69, 0x2f, 0x65, 0x64, 0x67,
+	0x65, 0x2f, 0x70, 0x62, 0x2f, 0x65, 0x64, 0x67, 0x65, 0x5f, 0x63, 0x74, 0x72, 0x6c, 0x5f, 0x70,
+	0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (
diff --git a/common/pb/edge_ctrl_pb/edge_ctrl.proto b/common/pb/edge_ctrl_pb/edge_ctrl.proto
index 58cbe9b77..0b971bcb5 100644
--- a/common/pb/edge_ctrl_pb/edge_ctrl.proto
+++ b/common/pb/edge_ctrl_pb/edge_ctrl.proto
@@ -244,6 +244,7 @@ enum CreateTerminatorResult {
   FailedIdConflict = 1;
   FailedOther = 2;
   FailedBusy = 3;
+  FailedInvalidSession = 4;
 }
 
 message CreateTerminatorV2Response {
diff --git a/common/pb/edge_mgmt_pb/edge_mgmt.pb.go b/common/pb/edge_mgmt_pb/edge_mgmt.pb.go
index ffd8fbb8c..b8fa212d6 100644
--- a/common/pb/edge_mgmt_pb/edge_mgmt.pb.go
+++ b/common/pb/edge_mgmt_pb/edge_mgmt.pb.go
@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.31.0
-// 	protoc        v4.23.4
+// 	protoc        v3.21.12
 // source: edge_mgmt.proto
 
 package edge_mgmt_pb
diff --git a/common/pb/mgmt_pb/impl.go b/common/pb/mgmt_pb/impl.go
index 2f6e08885..4ae9e8299 100644
--- a/common/pb/mgmt_pb/impl.go
+++ b/common/pb/mgmt_pb/impl.go
@@ -35,3 +35,15 @@ func (request *ValidateRouterLinksResponse) GetContentType() int32 {
 func (request *RouterLinkDetails) GetContentType() int32 {
 	return int32(ContentType_ValidateRouterLinksResultType)
 }
+
+func (request *ValidateRouterSdkTerminatorsRequest) GetContentType() int32 {
+	return int32(ContentType_ValidateRouterSdkTerminatorsRequestType)
+}
+
+func (request *ValidateRouterSdkTerminatorsResponse) GetContentType() int32 {
+	return int32(ContentType_ValidateRouterSdkTerminatorsResponseType)
+}
+
+func (request *RouterSdkTerminatorsDetails) GetContentType() int32 {
+	return int32(ContentType_ValidateRouterSdkTerminatorsResultType)
+}
diff --git a/common/pb/mgmt_pb/mgmt.pb.go b/common/pb/mgmt_pb/mgmt.pb.go
index 76c9a8b29..28210486c 100644
--- a/common/pb/mgmt_pb/mgmt.pb.go
+++ b/common/pb/mgmt_pb/mgmt.pb.go
@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.31.0
-// 	protoc        v4.23.4
+// 	protoc        v3.21.12
 // source: mgmt.proto
 
 package mgmt_pb
@@ -56,12 +56,15 @@ const (
 	ContentType_RaftTransferLeadershipRequestType ContentType = 10084
 	ContentType_RaftInitFromDb                    ContentType = 10085
 	// Validate
-	ContentType_ValidateTerminatorsRequestType  ContentType = 10100
-	ContentType_ValidateTerminatorResponseType  ContentType = 10101
-	ContentType_ValidateTerminatorResultType    ContentType = 10102
-	ContentType_ValidateRouterLinksRequestType  ContentType = 10103
-	ContentType_ValidateRouterLinksResponseType ContentType = 10104
-	ContentType_ValidateRouterLinksResultType   ContentType = 10105
+	ContentType_ValidateTerminatorsRequestType           ContentType = 10100
+	ContentType_ValidateTerminatorResponseType           ContentType = 10101
+	ContentType_ValidateTerminatorResultType             ContentType = 10102
+	ContentType_ValidateRouterLinksRequestType           ContentType = 10103
+	ContentType_ValidateRouterLinksResponseType          ContentType = 10104
+	ContentType_ValidateRouterLinksResultType            ContentType = 10105
+	ContentType_ValidateRouterSdkTerminatorsRequestType  ContentType = 10106
+	ContentType_ValidateRouterSdkTerminatorsResponseType ContentType = 10107
+	ContentType_ValidateRouterSdkTerminatorsResultType   ContentType = 10108
 )
 
 // Enum value maps for ContentType.
@@ -98,6 +101,9 @@ var (
 		10103: "ValidateRouterLinksRequestType",
 		10104: "ValidateRouterLinksResponseType",
 		10105: "ValidateRouterLinksResultType",
+		10106: "ValidateRouterSdkTerminatorsRequestType",
+		10107: "ValidateRouterSdkTerminatorsResponseType",
+		10108: "ValidateRouterSdkTerminatorsResultType",
 	}
 	ContentType_value = map[string]int32{
 		"Zero":                                      0,
@@ -131,6 +137,9 @@ var (
 		"ValidateRouterLinksRequestType":            10103,
 		"ValidateRouterLinksResponseType":           10104,
 		"ValidateRouterLinksResultType":             10105,
+		"ValidateRouterSdkTerminatorsRequestType":   10106,
+		"ValidateRouterSdkTerminatorsResponseType":  10107,
+		"ValidateRouterSdkTerminatorsResultType":    10108,
 	}
 )
 
@@ -1637,6 +1646,290 @@ func (x *RouterLinkDetail) GetDialed() bool {
 	return false
 }
 
+type ValidateRouterSdkTerminatorsRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Filter string `protobuf:"bytes,1,opt,name=filter,proto3" json:"filter,omitempty"`
+}
+
+func (x *ValidateRouterSdkTerminatorsRequest) Reset() {
+	*x = ValidateRouterSdkTerminatorsRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_mgmt_proto_msgTypes[17]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *ValidateRouterSdkTerminatorsRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ValidateRouterSdkTerminatorsRequest) ProtoMessage() {}
+
+func (x *ValidateRouterSdkTerminatorsRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_mgmt_proto_msgTypes[17]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ValidateRouterSdkTerminatorsRequest.ProtoReflect.Descriptor instead.
+func (*ValidateRouterSdkTerminatorsRequest) Descriptor() ([]byte, []int) {
+	return file_mgmt_proto_rawDescGZIP(), []int{17}
+}
+
+func (x *ValidateRouterSdkTerminatorsRequest) GetFilter() string {
+	if x != nil {
+		return x.Filter
+	}
+	return ""
+}
+
+type ValidateRouterSdkTerminatorsResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Success     bool   `protobuf:"varint,1,opt,name=success,proto3" json:"success,omitempty"`
+	Message     string `protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"`
+	RouterCount uint64 `protobuf:"varint,3,opt,name=routerCount,proto3" json:"routerCount,omitempty"`
+}
+
+func (x *ValidateRouterSdkTerminatorsResponse) Reset() {
+	*x = ValidateRouterSdkTerminatorsResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_mgmt_proto_msgTypes[18]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *ValidateRouterSdkTerminatorsResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ValidateRouterSdkTerminatorsResponse) ProtoMessage() {}
+
+func (x *ValidateRouterSdkTerminatorsResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_mgmt_proto_msgTypes[18]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ValidateRouterSdkTerminatorsResponse.ProtoReflect.Descriptor instead.
+func (*ValidateRouterSdkTerminatorsResponse) Descriptor() ([]byte, []int) {
+	return file_mgmt_proto_rawDescGZIP(), []int{18}
+}
+
+func (x *ValidateRouterSdkTerminatorsResponse) GetSuccess() bool {
+	if x != nil {
+		return x.Success
+	}
+	return false
+}
+
+func (x *ValidateRouterSdkTerminatorsResponse) GetMessage() string {
+	if x != nil {
+		return x.Message
+	}
+	return ""
+}
+
+func (x *ValidateRouterSdkTerminatorsResponse) GetRouterCount() uint64 {
+	if x != nil {
+		return x.RouterCount
+	}
+	return 0
+}
+
+type RouterSdkTerminatorsDetails struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	RouterId        string                       `protobuf:"bytes,1,opt,name=routerId,proto3" json:"routerId,omitempty"`
+	RouterName      string                       `protobuf:"bytes,2,opt,name=routerName,proto3" json:"routerName,omitempty"`
+	ValidateSuccess bool                         `protobuf:"varint,3,opt,name=validateSuccess,proto3" json:"validateSuccess,omitempty"`
+	Message         string                       `protobuf:"bytes,4,opt,name=message,proto3" json:"message,omitempty"`
+	Details         []*RouterSdkTerminatorDetail `protobuf:"bytes,5,rep,name=details,proto3" json:"details,omitempty"`
+}
+
+func (x *RouterSdkTerminatorsDetails) Reset() {
+	*x = RouterSdkTerminatorsDetails{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_mgmt_proto_msgTypes[19]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *RouterSdkTerminatorsDetails) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*RouterSdkTerminatorsDetails) ProtoMessage() {}
+
+func (x *RouterSdkTerminatorsDetails) ProtoReflect() protoreflect.Message {
+	mi := &file_mgmt_proto_msgTypes[19]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use RouterSdkTerminatorsDetails.ProtoReflect.Descriptor instead.
+func (*RouterSdkTerminatorsDetails) Descriptor() ([]byte, []int) {
+	return file_mgmt_proto_rawDescGZIP(), []int{19}
+}
+
+func (x *RouterSdkTerminatorsDetails) GetRouterId() string {
+	if x != nil {
+		return x.RouterId
+	}
+	return ""
+}
+
+func (x *RouterSdkTerminatorsDetails) GetRouterName() string {
+	if x != nil {
+		return x.RouterName
+	}
+	return ""
+}
+
+func (x *RouterSdkTerminatorsDetails) GetValidateSuccess() bool {
+	if x != nil {
+		return x.ValidateSuccess
+	}
+	return false
+}
+
+func (x *RouterSdkTerminatorsDetails) GetMessage() string {
+	if x != nil {
+		return x.Message
+	}
+	return ""
+}
+
+func (x *RouterSdkTerminatorsDetails) GetDetails() []*RouterSdkTerminatorDetail {
+	if x != nil {
+		return x.Details
+	}
+	return nil
+}
+
+type RouterSdkTerminatorDetail struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	TerminatorId     string          `protobuf:"bytes,1,opt,name=terminatorId,proto3" json:"terminatorId,omitempty"`
+	CtrlState        TerminatorState `protobuf:"varint,2,opt,name=ctrlState,proto3,enum=ziti.mgmt_pb.TerminatorState" json:"ctrlState,omitempty"`
+	RouterState      string          `protobuf:"bytes,3,opt,name=routerState,proto3" json:"routerState,omitempty"`
+	IsValid          bool            `protobuf:"varint,4,opt,name=isValid,proto3" json:"isValid,omitempty"`
+	OperaationActive bool            `protobuf:"varint,5,opt,name=operaationActive,proto3" json:"operaationActive,omitempty"`
+	CreateTime       string          `protobuf:"bytes,6,opt,name=createTime,proto3" json:"createTime,omitempty"`
+	LastAttempt      string          `protobuf:"bytes,7,opt,name=lastAttempt,proto3" json:"lastAttempt,omitempty"`
+}
+
+func (x *RouterSdkTerminatorDetail) Reset() {
+	*x = RouterSdkTerminatorDetail{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_mgmt_proto_msgTypes[20]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *RouterSdkTerminatorDetail) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*RouterSdkTerminatorDetail) ProtoMessage() {}
+
+func (x *RouterSdkTerminatorDetail) ProtoReflect() protoreflect.Message {
+	mi := &file_mgmt_proto_msgTypes[20]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use RouterSdkTerminatorDetail.ProtoReflect.Descriptor instead.
+func (*RouterSdkTerminatorDetail) Descriptor() ([]byte, []int) {
+	return file_mgmt_proto_rawDescGZIP(), []int{20}
+}
+
+func (x *RouterSdkTerminatorDetail) GetTerminatorId() string {
+	if x != nil {
+		return x.TerminatorId
+	}
+	return ""
+}
+
+func (x *RouterSdkTerminatorDetail) GetCtrlState() TerminatorState {
+	if x != nil {
+		return x.CtrlState
+	}
+	return TerminatorState_Valid
+}
+
+func (x *RouterSdkTerminatorDetail) GetRouterState() string {
+	if x != nil {
+		return x.RouterState
+	}
+	return ""
+}
+
+func (x *RouterSdkTerminatorDetail) GetIsValid() bool {
+	if x != nil {
+		return x.IsValid
+	}
+	return false
+}
+
+func (x *RouterSdkTerminatorDetail) GetOperaationActive() bool {
+	if x != nil {
+		return x.OperaationActive
+	}
+	return false
+}
+
+func (x *RouterSdkTerminatorDetail) GetCreateTime() string {
+	if x != nil {
+		return x.CreateTime
+	}
+	return ""
+}
+
+func (x *RouterSdkTerminatorDetail) GetLastAttempt() string {
+	if x != nil {
+		return x.LastAttempt
+	}
+	return ""
+}
+
 type StreamMetricsRequest_MetricMatcher struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
@@ -1649,7 +1942,7 @@ type StreamMetricsRequest_MetricMatcher struct {
 func (x *StreamMetricsRequest_MetricMatcher) Reset() {
 	*x = StreamMetricsRequest_MetricMatcher{}
 	if protoimpl.UnsafeEnabled {
-		mi := &file_mgmt_proto_msgTypes[17]
+		mi := &file_mgmt_proto_msgTypes[21]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		ms.StoreMessageInfo(mi)
 	}
@@ -1662,7 +1955,7 @@ func (x *StreamMetricsRequest_MetricMatcher) String() string {
 func (*StreamMetricsRequest_MetricMatcher) ProtoMessage() {}
 
 func (x *StreamMetricsRequest_MetricMatcher) ProtoReflect() protoreflect.Message {
-	mi := &file_mgmt_proto_msgTypes[17]
+	mi := &file_mgmt_proto_msgTypes[21]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -1706,7 +1999,7 @@ type StreamMetricsEvent_IntervalMetric struct {
 func (x *StreamMetricsEvent_IntervalMetric) Reset() {
 	*x = StreamMetricsEvent_IntervalMetric{}
 	if protoimpl.UnsafeEnabled {
-		mi := &file_mgmt_proto_msgTypes[21]
+		mi := &file_mgmt_proto_msgTypes[25]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		ms.StoreMessageInfo(mi)
 	}
@@ -1719,7 +2012,7 @@ func (x *StreamMetricsEvent_IntervalMetric) String() string {
 func (*StreamMetricsEvent_IntervalMetric) ProtoMessage() {}
 
 func (x *StreamMetricsEvent_IntervalMetric) ProtoReflect() protoreflect.Message {
-	mi := &file_mgmt_proto_msgTypes[21]
+	mi := &file_mgmt_proto_msgTypes[25]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -1776,7 +2069,7 @@ type InspectResponse_InspectValue struct {
 func (x *InspectResponse_InspectValue) Reset() {
 	*x = InspectResponse_InspectValue{}
 	if protoimpl.UnsafeEnabled {
-		mi := &file_mgmt_proto_msgTypes[24]
+		mi := &file_mgmt_proto_msgTypes[28]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		ms.StoreMessageInfo(mi)
 	}
@@ -1789,7 +2082,7 @@ func (x *InspectResponse_InspectValue) String() string {
 func (*InspectResponse_InspectValue) ProtoMessage() {}
 
 func (x *InspectResponse_InspectValue) ProtoReflect() protoreflect.Message {
-	mi := &file_mgmt_proto_msgTypes[24]
+	mi := &file_mgmt_proto_msgTypes[28]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -2071,104 +2364,157 @@ var file_mgmt_proto_rawDesc = []byte{
 	0x6f, 0x75, 0x74, 0x65, 0x72, 0x49, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x64,
 	0x65, 0x73, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x72, 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x64,
 	0x69, 0x61, 0x6c, 0x65, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x64, 0x69, 0x61,
-	0x6c, 0x65, 0x64, 0x2a, 0x95, 0x08, 0x0a, 0x0b, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x54,
-	0x79, 0x70, 0x65, 0x12, 0x08, 0x0a, 0x04, 0x5a, 0x65, 0x72, 0x6f, 0x10, 0x00, 0x12, 0x1c, 0x0a,
-	0x17, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71,
-	0x75, 0x65, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10, 0xb8, 0x4e, 0x12, 0x1a, 0x0a, 0x15, 0x53,
-	0x74, 0x72, 0x65, 0x61, 0x6d, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x45, 0x76, 0x65, 0x6e, 0x74,
-	0x54, 0x79, 0x70, 0x65, 0x10, 0xb9, 0x4e, 0x12, 0x20, 0x0a, 0x1b, 0x54, 0x6f, 0x67, 0x67, 0x6c,
-	0x65, 0x50, 0x69, 0x70, 0x65, 0x54, 0x72, 0x61, 0x63, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65,
-	0x73, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10, 0xbc, 0x4e, 0x12, 0x23, 0x0a, 0x1e, 0x54, 0x6f, 0x67,
-	0x67, 0x6c, 0x65, 0x43, 0x69, 0x72, 0x63, 0x75, 0x69, 0x74, 0x54, 0x72, 0x61, 0x63, 0x65, 0x73,
-	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10, 0xbd, 0x4e, 0x12, 0x1c,
-	0x0a, 0x17, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x54, 0x72, 0x61, 0x63, 0x65, 0x73, 0x52, 0x65,
-	0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10, 0xbe, 0x4e, 0x12, 0x1a, 0x0a, 0x15,
-	0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x54, 0x72, 0x61, 0x63, 0x65, 0x73, 0x45, 0x76, 0x65, 0x6e,
-	0x74, 0x54, 0x79, 0x70, 0x65, 0x10, 0xbf, 0x4e, 0x12, 0x17, 0x0a, 0x12, 0x49, 0x6e, 0x73, 0x70,
-	0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10, 0xc0,
-	0x4e, 0x12, 0x18, 0x0a, 0x13, 0x49, 0x6e, 0x73, 0x70, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70,
-	0x6f, 0x6e, 0x73, 0x65, 0x54, 0x79, 0x70, 0x65, 0x10, 0xc1, 0x4e, 0x12, 0x1a, 0x0a, 0x15, 0x53,
-	0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x44, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
-	0x54, 0x79, 0x70, 0x65, 0x10, 0xd6, 0x4e, 0x12, 0x25, 0x0a, 0x20, 0x52, 0x6f, 0x75, 0x74, 0x65,
-	0x72, 0x44, 0x65, 0x62, 0x75, 0x67, 0x46, 0x6f, 0x72, 0x67, 0x65, 0x74, 0x4c, 0x69, 0x6e, 0x6b,
-	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10, 0xd7, 0x4e, 0x12, 0x2c,
-	0x0a, 0x27, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x72, 0x44, 0x65, 0x62, 0x75, 0x67, 0x54, 0x6f, 0x67,
-	0x67, 0x6c, 0x65, 0x43, 0x74, 0x72, 0x6c, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x52, 0x65,
-	0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10, 0xd8, 0x4e, 0x12, 0x26, 0x0a, 0x21,
-	0x52, 0x6f, 0x75, 0x74, 0x65, 0x72, 0x44, 0x65, 0x62, 0x75, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74,
-	0x65, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x79, 0x70,
-	0x65, 0x10, 0xd9, 0x4e, 0x12, 0x2e, 0x0a, 0x29, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x72, 0x44, 0x65,
-	0x62, 0x75, 0x67, 0x44, 0x75, 0x6d, 0x70, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x65, 0x72,
-	0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x79, 0x70,
-	0x65, 0x10, 0xda, 0x4e, 0x12, 0x24, 0x0a, 0x1f, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x72, 0x44, 0x65,
-	0x62, 0x75, 0x67, 0x44, 0x75, 0x6d, 0x70, 0x4c, 0x69, 0x6e, 0x6b, 0x73, 0x52, 0x65, 0x71, 0x75,
-	0x65, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10, 0xdb, 0x4e, 0x12, 0x22, 0x0a, 0x1d, 0x52, 0x6f,
-	0x75, 0x74, 0x65, 0x72, 0x44, 0x65, 0x62, 0x75, 0x67, 0x55, 0x6e, 0x72, 0x6f, 0x75, 0x74, 0x65,
-	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10, 0xdc, 0x4e, 0x12, 0x1d,
-	0x0a, 0x18, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x72, 0x51, 0x75, 0x69, 0x65, 0x73, 0x63, 0x65, 0x52,
-	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10, 0xdd, 0x4e, 0x12, 0x1f, 0x0a,
-	0x1a, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x72, 0x44, 0x65, 0x71, 0x75, 0x69, 0x65, 0x73, 0x63, 0x65,
-	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10, 0xde, 0x4e, 0x12, 0x22,
-	0x0a, 0x1d, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x72, 0x44, 0x65, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x73,
-	0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10,
-	0xdf, 0x4e, 0x12, 0x1f, 0x0a, 0x1a, 0x52, 0x61, 0x66, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x4d, 0x65,
-	0x6d, 0x62, 0x65, 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65,
-	0x10, 0xe0, 0x4e, 0x12, 0x20, 0x0a, 0x1b, 0x52, 0x61, 0x66, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x4d,
-	0x65, 0x6d, 0x62, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x54, 0x79,
-	0x70, 0x65, 0x10, 0xe1, 0x4e, 0x12, 0x1b, 0x0a, 0x16, 0x52, 0x61, 0x66, 0x74, 0x41, 0x64, 0x64,
-	0x50, 0x65, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10,
-	0xe2, 0x4e, 0x12, 0x1e, 0x0a, 0x19, 0x52, 0x61, 0x66, 0x74, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65,
-	0x50, 0x65, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10,
-	0xe3, 0x4e, 0x12, 0x26, 0x0a, 0x21, 0x52, 0x61, 0x66, 0x74, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66,
-	0x65, 0x72, 0x4c, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x68, 0x69, 0x70, 0x52, 0x65, 0x71, 0x75,
-	0x65, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10, 0xe4, 0x4e, 0x12, 0x13, 0x0a, 0x0e, 0x52, 0x61,
-	0x66, 0x74, 0x49, 0x6e, 0x69, 0x74, 0x46, 0x72, 0x6f, 0x6d, 0x44, 0x62, 0x10, 0xe5, 0x4e, 0x12,
-	0x23, 0x0a, 0x1e, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x65, 0x72, 0x6d, 0x69,
-	0x6e, 0x61, 0x74, 0x6f, 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x79, 0x70,
-	0x65, 0x10, 0xf4, 0x4e, 0x12, 0x23, 0x0a, 0x1e, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65,
-	0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
-	0x73, 0x65, 0x54, 0x79, 0x70, 0x65, 0x10, 0xf5, 0x4e, 0x12, 0x21, 0x0a, 0x1c, 0x56, 0x61, 0x6c,
-	0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, 0x52,
-	0x65, 0x73, 0x75, 0x6c, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10, 0xf6, 0x4e, 0x12, 0x23, 0x0a, 0x1e,
-	0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x72, 0x4c, 0x69,
-	0x6e, 0x6b, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10, 0xf7,
-	0x4e, 0x12, 0x24, 0x0a, 0x1f, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x52, 0x6f, 0x75,
-	0x74, 0x65, 0x72, 0x4c, 0x69, 0x6e, 0x6b, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
-	0x54, 0x79, 0x70, 0x65, 0x10, 0xf8, 0x4e, 0x12, 0x22, 0x0a, 0x1d, 0x56, 0x61, 0x6c, 0x69, 0x64,
-	0x61, 0x74, 0x65, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x72, 0x4c, 0x69, 0x6e, 0x6b, 0x73, 0x52, 0x65,
-	0x73, 0x75, 0x6c, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10, 0xf9, 0x4e, 0x2a, 0x53, 0x0a, 0x06, 0x48,
-	0x65, 0x61, 0x64, 0x65, 0x72, 0x12, 0x0e, 0x0a, 0x0a, 0x4e, 0x6f, 0x6e, 0x65, 0x48, 0x65, 0x61,
-	0x64, 0x65, 0x72, 0x10, 0x00, 0x12, 0x13, 0x0a, 0x0f, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79,
-	0x70, 0x65, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x10, 0x0a, 0x12, 0x12, 0x0a, 0x0e, 0x43, 0x74,
-	0x72, 0x6c, 0x43, 0x68, 0x61, 0x6e, 0x54, 0x6f, 0x67, 0x67, 0x6c, 0x65, 0x10, 0x0b, 0x12, 0x10,
-	0x0a, 0x0c, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x49, 0x64, 0x10, 0x0c,
-	0x2a, 0x78, 0x0a, 0x16, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x69, 0x72, 0x63, 0x75, 0x69,
-	0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x0e, 0x43, 0x69,
-	0x72, 0x63, 0x75, 0x69, 0x74, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x10, 0x00, 0x12, 0x12,
-	0x0a, 0x0e, 0x43, 0x69, 0x72, 0x63, 0x75, 0x69, 0x74, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x64,
-	0x10, 0x01, 0x12, 0x12, 0x0a, 0x0e, 0x43, 0x69, 0x72, 0x63, 0x75, 0x69, 0x74, 0x50, 0x72, 0x65,
-	0x73, 0x65, 0x6e, 0x74, 0x10, 0x02, 0x12, 0x0f, 0x0a, 0x0b, 0x50, 0x61, 0x74, 0x68, 0x55, 0x70,
-	0x64, 0x61, 0x74, 0x65, 0x64, 0x10, 0x03, 0x12, 0x11, 0x0a, 0x0d, 0x43, 0x69, 0x72, 0x63, 0x75,
-	0x69, 0x74, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x10, 0x04, 0x2a, 0x2b, 0x0a, 0x0f, 0x54, 0x72,
-	0x61, 0x63, 0x65, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a,
-	0x07, 0x45, 0x58, 0x43, 0x4c, 0x55, 0x44, 0x45, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x49, 0x4e,
-	0x43, 0x4c, 0x55, 0x44, 0x45, 0x10, 0x01, 0x2a, 0x77, 0x0a, 0x0f, 0x54, 0x65, 0x72, 0x6d, 0x69,
-	0x6e, 0x61, 0x74, 0x6f, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x09, 0x0a, 0x05, 0x56, 0x61,
-	0x6c, 0x69, 0x64, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e,
-	0x10, 0x01, 0x12, 0x19, 0x0a, 0x15, 0x49, 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x55, 0x6e, 0x6b,
-	0x6e, 0x6f, 0x77, 0x6e, 0x42, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x10, 0x02, 0x12, 0x1c, 0x0a,
-	0x18, 0x49, 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x55, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x54,
-	0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, 0x10, 0x03, 0x12, 0x13, 0x0a, 0x0f, 0x49,
-	0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x42, 0x61, 0x64, 0x53, 0x74, 0x61, 0x74, 0x65, 0x10, 0x04,
-	0x2a, 0x53, 0x0a, 0x09, 0x4c, 0x69, 0x6e, 0x6b, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0f, 0x0a,
-	0x0b, 0x4c, 0x69, 0x6e, 0x6b, 0x55, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x10, 0x00, 0x12, 0x13,
-	0x0a, 0x0f, 0x4c, 0x69, 0x6e, 0x6b, 0x45, 0x73, 0x74, 0x61, 0x62, 0x6c, 0x69, 0x73, 0x68, 0x65,
-	0x64, 0x10, 0x01, 0x12, 0x0f, 0x0a, 0x0b, 0x4c, 0x69, 0x6e, 0x6b, 0x50, 0x65, 0x6e, 0x64, 0x69,
-	0x6e, 0x67, 0x10, 0x02, 0x12, 0x0f, 0x0a, 0x0b, 0x4c, 0x69, 0x6e, 0x6b, 0x44, 0x69, 0x61, 0x6c,
-	0x69, 0x6e, 0x67, 0x10, 0x03, 0x42, 0x27, 0x5a, 0x25, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e,
-	0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x7a, 0x69, 0x74, 0x69, 0x2f, 0x66, 0x61, 0x62,
-	0x72, 0x69, 0x63, 0x2f, 0x70, 0x62, 0x2f, 0x6d, 0x67, 0x6d, 0x74, 0x5f, 0x70, 0x62, 0x62, 0x06,
-	0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x6c, 0x65, 0x64, 0x22, 0x3d, 0x0a, 0x23, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x52,
+	0x6f, 0x75, 0x74, 0x65, 0x72, 0x53, 0x64, 0x6b, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74,
+	0x6f, 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x66, 0x69,
+	0x6c, 0x74, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x66, 0x69, 0x6c, 0x74,
+	0x65, 0x72, 0x22, 0x7c, 0x0a, 0x24, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x52, 0x6f,
+	0x75, 0x74, 0x65, 0x72, 0x53, 0x64, 0x6b, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x6f,
+	0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x75,
+	0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x73, 0x75, 0x63,
+	0x63, 0x65, 0x73, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18,
+	0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x20,
+	0x0a, 0x0b, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x03, 0x20,
+	0x01, 0x28, 0x04, 0x52, 0x0b, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x75, 0x6e, 0x74,
+	0x22, 0xe0, 0x01, 0x0a, 0x1b, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x72, 0x53, 0x64, 0x6b, 0x54, 0x65,
+	0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, 0x73, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73,
+	0x12, 0x1a, 0x0a, 0x08, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x72, 0x49, 0x64, 0x18, 0x01, 0x20, 0x01,
+	0x28, 0x09, 0x52, 0x08, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x72, 0x49, 0x64, 0x12, 0x1e, 0x0a, 0x0a,
+	0x72, 0x6f, 0x75, 0x74, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09,
+	0x52, 0x0a, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x28, 0x0a, 0x0f,
+	0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18,
+	0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x53,
+	0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67,
+	0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65,
+	0x12, 0x41, 0x0a, 0x07, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28,
+	0x0b, 0x32, 0x27, 0x2e, 0x7a, 0x69, 0x74, 0x69, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x5f, 0x70, 0x62,
+	0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x72, 0x53, 0x64, 0x6b, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e,
+	0x61, 0x74, 0x6f, 0x72, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x52, 0x07, 0x64, 0x65, 0x74, 0x61,
+	0x69, 0x6c, 0x73, 0x22, 0xa6, 0x02, 0x0a, 0x19, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x72, 0x53, 0x64,
+	0x6b, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, 0x44, 0x65, 0x74, 0x61, 0x69,
+	0x6c, 0x12, 0x22, 0x0a, 0x0c, 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, 0x49,
+	0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61,
+	0x74, 0x6f, 0x72, 0x49, 0x64, 0x12, 0x3b, 0x0a, 0x09, 0x63, 0x74, 0x72, 0x6c, 0x53, 0x74, 0x61,
+	0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1d, 0x2e, 0x7a, 0x69, 0x74, 0x69, 0x2e,
+	0x6d, 0x67, 0x6d, 0x74, 0x5f, 0x70, 0x62, 0x2e, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74,
+	0x6f, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x09, 0x63, 0x74, 0x72, 0x6c, 0x53, 0x74, 0x61,
+	0x74, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74,
+	0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x72, 0x53,
+	0x74, 0x61, 0x74, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x69, 0x73, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x18,
+	0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x69, 0x73, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x12, 0x2a,
+	0x0a, 0x10, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x63, 0x74, 0x69,
+	0x76, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x61,
+	0x74, 0x69, 0x6f, 0x6e, 0x41, 0x63, 0x74, 0x69, 0x76, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x72,
+	0x65, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a,
+	0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x6c, 0x61,
+	0x73, 0x74, 0x41, 0x74, 0x74, 0x65, 0x6d, 0x70, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52,
+	0x0b, 0x6c, 0x61, 0x73, 0x74, 0x41, 0x74, 0x74, 0x65, 0x6d, 0x70, 0x74, 0x2a, 0x9f, 0x09, 0x0a,
+	0x0b, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x08, 0x0a, 0x04,
+	0x5a, 0x65, 0x72, 0x6f, 0x10, 0x00, 0x12, 0x1c, 0x0a, 0x17, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d,
+	0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x79, 0x70,
+	0x65, 0x10, 0xb8, 0x4e, 0x12, 0x1a, 0x0a, 0x15, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x45, 0x76,
+	0x65, 0x6e, 0x74, 0x73, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10, 0xb9, 0x4e,
+	0x12, 0x20, 0x0a, 0x1b, 0x54, 0x6f, 0x67, 0x67, 0x6c, 0x65, 0x50, 0x69, 0x70, 0x65, 0x54, 0x72,
+	0x61, 0x63, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10,
+	0xbc, 0x4e, 0x12, 0x23, 0x0a, 0x1e, 0x54, 0x6f, 0x67, 0x67, 0x6c, 0x65, 0x43, 0x69, 0x72, 0x63,
+	0x75, 0x69, 0x74, 0x54, 0x72, 0x61, 0x63, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
+	0x54, 0x79, 0x70, 0x65, 0x10, 0xbd, 0x4e, 0x12, 0x1c, 0x0a, 0x17, 0x53, 0x74, 0x72, 0x65, 0x61,
+	0x6d, 0x54, 0x72, 0x61, 0x63, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x79,
+	0x70, 0x65, 0x10, 0xbe, 0x4e, 0x12, 0x1a, 0x0a, 0x15, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x54,
+	0x72, 0x61, 0x63, 0x65, 0x73, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10, 0xbf,
+	0x4e, 0x12, 0x17, 0x0a, 0x12, 0x49, 0x6e, 0x73, 0x70, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75,
+	0x65, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10, 0xc0, 0x4e, 0x12, 0x18, 0x0a, 0x13, 0x49, 0x6e,
+	0x73, 0x70, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x54, 0x79, 0x70,
+	0x65, 0x10, 0xc1, 0x4e, 0x12, 0x1a, 0x0a, 0x15, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74,
+	0x44, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10, 0xd6, 0x4e,
+	0x12, 0x25, 0x0a, 0x20, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x72, 0x44, 0x65, 0x62, 0x75, 0x67, 0x46,
+	0x6f, 0x72, 0x67, 0x65, 0x74, 0x4c, 0x69, 0x6e, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
+	0x54, 0x79, 0x70, 0x65, 0x10, 0xd7, 0x4e, 0x12, 0x2c, 0x0a, 0x27, 0x52, 0x6f, 0x75, 0x74, 0x65,
+	0x72, 0x44, 0x65, 0x62, 0x75, 0x67, 0x54, 0x6f, 0x67, 0x67, 0x6c, 0x65, 0x43, 0x74, 0x72, 0x6c,
+	0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x79,
+	0x70, 0x65, 0x10, 0xd8, 0x4e, 0x12, 0x26, 0x0a, 0x21, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x72, 0x44,
+	0x65, 0x62, 0x75, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x52,
+	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10, 0xd9, 0x4e, 0x12, 0x2e, 0x0a,
+	0x29, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x72, 0x44, 0x65, 0x62, 0x75, 0x67, 0x44, 0x75, 0x6d, 0x70,
+	0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x65, 0x72, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x52,
+	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10, 0xda, 0x4e, 0x12, 0x24, 0x0a,
+	0x1f, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x72, 0x44, 0x65, 0x62, 0x75, 0x67, 0x44, 0x75, 0x6d, 0x70,
+	0x4c, 0x69, 0x6e, 0x6b, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65,
+	0x10, 0xdb, 0x4e, 0x12, 0x22, 0x0a, 0x1d, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x72, 0x44, 0x65, 0x62,
+	0x75, 0x67, 0x55, 0x6e, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
+	0x54, 0x79, 0x70, 0x65, 0x10, 0xdc, 0x4e, 0x12, 0x1d, 0x0a, 0x18, 0x52, 0x6f, 0x75, 0x74, 0x65,
+	0x72, 0x51, 0x75, 0x69, 0x65, 0x73, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x54,
+	0x79, 0x70, 0x65, 0x10, 0xdd, 0x4e, 0x12, 0x1f, 0x0a, 0x1a, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x72,
+	0x44, 0x65, 0x71, 0x75, 0x69, 0x65, 0x73, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
+	0x54, 0x79, 0x70, 0x65, 0x10, 0xde, 0x4e, 0x12, 0x22, 0x0a, 0x1d, 0x52, 0x6f, 0x75, 0x74, 0x65,
+	0x72, 0x44, 0x65, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71,
+	0x75, 0x65, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10, 0xdf, 0x4e, 0x12, 0x1f, 0x0a, 0x1a, 0x52,
+	0x61, 0x66, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x4d, 0x65, 0x6d, 0x62, 0x65, 0x72, 0x73, 0x52, 0x65,
+	0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10, 0xe0, 0x4e, 0x12, 0x20, 0x0a, 0x1b,
+	0x52, 0x61, 0x66, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x4d, 0x65, 0x6d, 0x62, 0x65, 0x72, 0x73, 0x52,
+	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x54, 0x79, 0x70, 0x65, 0x10, 0xe1, 0x4e, 0x12, 0x1b,
+	0x0a, 0x16, 0x52, 0x61, 0x66, 0x74, 0x41, 0x64, 0x64, 0x50, 0x65, 0x65, 0x72, 0x52, 0x65, 0x71,
+	0x75, 0x65, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10, 0xe2, 0x4e, 0x12, 0x1e, 0x0a, 0x19, 0x52,
+	0x61, 0x66, 0x74, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x50, 0x65, 0x65, 0x72, 0x52, 0x65, 0x71,
+	0x75, 0x65, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10, 0xe3, 0x4e, 0x12, 0x26, 0x0a, 0x21, 0x52,
+	0x61, 0x66, 0x74, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x4c, 0x65, 0x61, 0x64, 0x65,
+	0x72, 0x73, 0x68, 0x69, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65,
+	0x10, 0xe4, 0x4e, 0x12, 0x13, 0x0a, 0x0e, 0x52, 0x61, 0x66, 0x74, 0x49, 0x6e, 0x69, 0x74, 0x46,
+	0x72, 0x6f, 0x6d, 0x44, 0x62, 0x10, 0xe5, 0x4e, 0x12, 0x23, 0x0a, 0x1e, 0x56, 0x61, 0x6c, 0x69,
+	0x64, 0x61, 0x74, 0x65, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, 0x73, 0x52,
+	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10, 0xf4, 0x4e, 0x12, 0x23, 0x0a,
+	0x1e, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61,
+	0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x54, 0x79, 0x70, 0x65, 0x10,
+	0xf5, 0x4e, 0x12, 0x21, 0x0a, 0x1c, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x65,
+	0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x54, 0x79,
+	0x70, 0x65, 0x10, 0xf6, 0x4e, 0x12, 0x23, 0x0a, 0x1e, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74,
+	0x65, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x72, 0x4c, 0x69, 0x6e, 0x6b, 0x73, 0x52, 0x65, 0x71, 0x75,
+	0x65, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10, 0xf7, 0x4e, 0x12, 0x24, 0x0a, 0x1f, 0x56, 0x61,
+	0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x72, 0x4c, 0x69, 0x6e, 0x6b,
+	0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x54, 0x79, 0x70, 0x65, 0x10, 0xf8, 0x4e,
+	0x12, 0x22, 0x0a, 0x1d, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x52, 0x6f, 0x75, 0x74,
+	0x65, 0x72, 0x4c, 0x69, 0x6e, 0x6b, 0x73, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x54, 0x79, 0x70,
+	0x65, 0x10, 0xf9, 0x4e, 0x12, 0x2c, 0x0a, 0x27, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65,
+	0x52, 0x6f, 0x75, 0x74, 0x65, 0x72, 0x53, 0x64, 0x6b, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61,
+	0x74, 0x6f, 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10,
+	0xfa, 0x4e, 0x12, 0x2d, 0x0a, 0x28, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x52, 0x6f,
+	0x75, 0x74, 0x65, 0x72, 0x53, 0x64, 0x6b, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x6f,
+	0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x54, 0x79, 0x70, 0x65, 0x10, 0xfb,
+	0x4e, 0x12, 0x2b, 0x0a, 0x26, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x52, 0x6f, 0x75,
+	0x74, 0x65, 0x72, 0x53, 0x64, 0x6b, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72,
+	0x73, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10, 0xfc, 0x4e, 0x2a, 0x53,
+	0x0a, 0x06, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x12, 0x0e, 0x0a, 0x0a, 0x4e, 0x6f, 0x6e, 0x65,
+	0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x10, 0x00, 0x12, 0x13, 0x0a, 0x0f, 0x45, 0x76, 0x65, 0x6e,
+	0x74, 0x54, 0x79, 0x70, 0x65, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x10, 0x0a, 0x12, 0x12, 0x0a,
+	0x0e, 0x43, 0x74, 0x72, 0x6c, 0x43, 0x68, 0x61, 0x6e, 0x54, 0x6f, 0x67, 0x67, 0x6c, 0x65, 0x10,
+	0x0b, 0x12, 0x10, 0x0a, 0x0c, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x49,
+	0x64, 0x10, 0x0c, 0x2a, 0x78, 0x0a, 0x16, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x69, 0x72,
+	0x63, 0x75, 0x69, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a,
+	0x0e, 0x43, 0x69, 0x72, 0x63, 0x75, 0x69, 0x74, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x10,
+	0x00, 0x12, 0x12, 0x0a, 0x0e, 0x43, 0x69, 0x72, 0x63, 0x75, 0x69, 0x74, 0x44, 0x65, 0x6c, 0x65,
+	0x74, 0x65, 0x64, 0x10, 0x01, 0x12, 0x12, 0x0a, 0x0e, 0x43, 0x69, 0x72, 0x63, 0x75, 0x69, 0x74,
+	0x50, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x10, 0x02, 0x12, 0x0f, 0x0a, 0x0b, 0x50, 0x61, 0x74,
+	0x68, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x10, 0x03, 0x12, 0x11, 0x0a, 0x0d, 0x43, 0x69,
+	0x72, 0x63, 0x75, 0x69, 0x74, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x10, 0x04, 0x2a, 0x2b, 0x0a,
+	0x0f, 0x54, 0x72, 0x61, 0x63, 0x65, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65,
+	0x12, 0x0b, 0x0a, 0x07, 0x45, 0x58, 0x43, 0x4c, 0x55, 0x44, 0x45, 0x10, 0x00, 0x12, 0x0b, 0x0a,
+	0x07, 0x49, 0x4e, 0x43, 0x4c, 0x55, 0x44, 0x45, 0x10, 0x01, 0x2a, 0x77, 0x0a, 0x0f, 0x54, 0x65,
+	0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x09, 0x0a,
+	0x05, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x6e, 0x6b, 0x6e,
+	0x6f, 0x77, 0x6e, 0x10, 0x01, 0x12, 0x19, 0x0a, 0x15, 0x49, 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64,
+	0x55, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x42, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x10, 0x02,
+	0x12, 0x1c, 0x0a, 0x18, 0x49, 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x55, 0x6e, 0x6b, 0x6e, 0x6f,
+	0x77, 0x6e, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, 0x10, 0x03, 0x12, 0x13,
+	0x0a, 0x0f, 0x49, 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x42, 0x61, 0x64, 0x53, 0x74, 0x61, 0x74,
+	0x65, 0x10, 0x04, 0x2a, 0x53, 0x0a, 0x09, 0x4c, 0x69, 0x6e, 0x6b, 0x53, 0x74, 0x61, 0x74, 0x65,
+	0x12, 0x0f, 0x0a, 0x0b, 0x4c, 0x69, 0x6e, 0x6b, 0x55, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x10,
+	0x00, 0x12, 0x13, 0x0a, 0x0f, 0x4c, 0x69, 0x6e, 0x6b, 0x45, 0x73, 0x74, 0x61, 0x62, 0x6c, 0x69,
+	0x73, 0x68, 0x65, 0x64, 0x10, 0x01, 0x12, 0x0f, 0x0a, 0x0b, 0x4c, 0x69, 0x6e, 0x6b, 0x50, 0x65,
+	0x6e, 0x64, 0x69, 0x6e, 0x67, 0x10, 0x02, 0x12, 0x0f, 0x0a, 0x0b, 0x4c, 0x69, 0x6e, 0x6b, 0x44,
+	0x69, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0x10, 0x03, 0x42, 0x27, 0x5a, 0x25, 0x67, 0x69, 0x74, 0x68,
+	0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x7a, 0x69, 0x74, 0x69, 0x2f,
+	0x66, 0x61, 0x62, 0x72, 0x69, 0x63, 0x2f, 0x70, 0x62, 0x2f, 0x6d, 0x67, 0x6d, 0x74, 0x5f, 0x70,
+	0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (
@@ -2184,66 +2530,72 @@ func file_mgmt_proto_rawDescGZIP() []byte {
 }
 
 var file_mgmt_proto_enumTypes = make([]protoimpl.EnumInfo, 6)
-var file_mgmt_proto_msgTypes = make([]protoimpl.MessageInfo, 25)
+var file_mgmt_proto_msgTypes = make([]protoimpl.MessageInfo, 29)
 var file_mgmt_proto_goTypes = []interface{}{
-	(ContentType)(0),                           // 0: ziti.mgmt_pb.ContentType
-	(Header)(0),                                // 1: ziti.mgmt_pb.Header
-	(StreamCircuitEventType)(0),                // 2: ziti.mgmt_pb.StreamCircuitEventType
-	(TraceFilterType)(0),                       // 3: ziti.mgmt_pb.TraceFilterType
-	(TerminatorState)(0),                       // 4: ziti.mgmt_pb.TerminatorState
-	(LinkState)(0),                             // 5: ziti.mgmt_pb.LinkState
-	(*StreamMetricsRequest)(nil),               // 6: ziti.mgmt_pb.StreamMetricsRequest
-	(*StreamMetricsEvent)(nil),                 // 7: ziti.mgmt_pb.StreamMetricsEvent
-	(*Path)(nil),                               // 8: ziti.mgmt_pb.Path
-	(*StreamCircuitsEvent)(nil),                // 9: ziti.mgmt_pb.StreamCircuitsEvent
-	(*ToggleCircuitTracesRequest)(nil),         // 10: ziti.mgmt_pb.ToggleCircuitTracesRequest
-	(*StreamTracesRequest)(nil),                // 11: ziti.mgmt_pb.StreamTracesRequest
-	(*InspectRequest)(nil),                     // 12: ziti.mgmt_pb.InspectRequest
-	(*InspectResponse)(nil),                    // 13: ziti.mgmt_pb.InspectResponse
-	(*RaftMember)(nil),                         // 14: ziti.mgmt_pb.RaftMember
-	(*RaftMemberListResponse)(nil),             // 15: ziti.mgmt_pb.RaftMemberListResponse
-	(*ValidateTerminatorsRequest)(nil),         // 16: ziti.mgmt_pb.ValidateTerminatorsRequest
-	(*ValidateTerminatorsResponse)(nil),        // 17: ziti.mgmt_pb.ValidateTerminatorsResponse
-	(*TerminatorDetail)(nil),                   // 18: ziti.mgmt_pb.TerminatorDetail
-	(*ValidateRouterLinksRequest)(nil),         // 19: ziti.mgmt_pb.ValidateRouterLinksRequest
-	(*ValidateRouterLinksResponse)(nil),        // 20: ziti.mgmt_pb.ValidateRouterLinksResponse
-	(*RouterLinkDetails)(nil),                  // 21: ziti.mgmt_pb.RouterLinkDetails
-	(*RouterLinkDetail)(nil),                   // 22: ziti.mgmt_pb.RouterLinkDetail
-	(*StreamMetricsRequest_MetricMatcher)(nil), // 23: ziti.mgmt_pb.StreamMetricsRequest.MetricMatcher
-	nil, // 24: ziti.mgmt_pb.StreamMetricsEvent.TagsEntry
-	nil, // 25: ziti.mgmt_pb.StreamMetricsEvent.IntMetricsEntry
-	nil, // 26: ziti.mgmt_pb.StreamMetricsEvent.FloatMetricsEntry
-	(*StreamMetricsEvent_IntervalMetric)(nil), // 27: ziti.mgmt_pb.StreamMetricsEvent.IntervalMetric
-	nil,                                  // 28: ziti.mgmt_pb.StreamMetricsEvent.MetricGroupEntry
-	nil,                                  // 29: ziti.mgmt_pb.StreamMetricsEvent.IntervalMetric.ValuesEntry
-	(*InspectResponse_InspectValue)(nil), // 30: ziti.mgmt_pb.InspectResponse.InspectValue
-	(*timestamppb.Timestamp)(nil),        // 31: google.protobuf.Timestamp
+	(ContentType)(0),                             // 0: ziti.mgmt_pb.ContentType
+	(Header)(0),                                  // 1: ziti.mgmt_pb.Header
+	(StreamCircuitEventType)(0),                  // 2: ziti.mgmt_pb.StreamCircuitEventType
+	(TraceFilterType)(0),                         // 3: ziti.mgmt_pb.TraceFilterType
+	(TerminatorState)(0),                         // 4: ziti.mgmt_pb.TerminatorState
+	(LinkState)(0),                               // 5: ziti.mgmt_pb.LinkState
+	(*StreamMetricsRequest)(nil),                 // 6: ziti.mgmt_pb.StreamMetricsRequest
+	(*StreamMetricsEvent)(nil),                   // 7: ziti.mgmt_pb.StreamMetricsEvent
+	(*Path)(nil),                                 // 8: ziti.mgmt_pb.Path
+	(*StreamCircuitsEvent)(nil),                  // 9: ziti.mgmt_pb.StreamCircuitsEvent
+	(*ToggleCircuitTracesRequest)(nil),           // 10: ziti.mgmt_pb.ToggleCircuitTracesRequest
+	(*StreamTracesRequest)(nil),                  // 11: ziti.mgmt_pb.StreamTracesRequest
+	(*InspectRequest)(nil),                       // 12: ziti.mgmt_pb.InspectRequest
+	(*InspectResponse)(nil),                      // 13: ziti.mgmt_pb.InspectResponse
+	(*RaftMember)(nil),                           // 14: ziti.mgmt_pb.RaftMember
+	(*RaftMemberListResponse)(nil),               // 15: ziti.mgmt_pb.RaftMemberListResponse
+	(*ValidateTerminatorsRequest)(nil),           // 16: ziti.mgmt_pb.ValidateTerminatorsRequest
+	(*ValidateTerminatorsResponse)(nil),          // 17: ziti.mgmt_pb.ValidateTerminatorsResponse
+	(*TerminatorDetail)(nil),                     // 18: ziti.mgmt_pb.TerminatorDetail
+	(*ValidateRouterLinksRequest)(nil),           // 19: ziti.mgmt_pb.ValidateRouterLinksRequest
+	(*ValidateRouterLinksResponse)(nil),          // 20: ziti.mgmt_pb.ValidateRouterLinksResponse
+	(*RouterLinkDetails)(nil),                    // 21: ziti.mgmt_pb.RouterLinkDetails
+	(*RouterLinkDetail)(nil),                     // 22: ziti.mgmt_pb.RouterLinkDetail
+	(*ValidateRouterSdkTerminatorsRequest)(nil),  // 23: ziti.mgmt_pb.ValidateRouterSdkTerminatorsRequest
+	(*ValidateRouterSdkTerminatorsResponse)(nil), // 24: ziti.mgmt_pb.ValidateRouterSdkTerminatorsResponse
+	(*RouterSdkTerminatorsDetails)(nil),          // 25: ziti.mgmt_pb.RouterSdkTerminatorsDetails
+	(*RouterSdkTerminatorDetail)(nil),            // 26: ziti.mgmt_pb.RouterSdkTerminatorDetail
+	(*StreamMetricsRequest_MetricMatcher)(nil),   // 27: ziti.mgmt_pb.StreamMetricsRequest.MetricMatcher
+	nil, // 28: ziti.mgmt_pb.StreamMetricsEvent.TagsEntry
+	nil, // 29: ziti.mgmt_pb.StreamMetricsEvent.IntMetricsEntry
+	nil, // 30: ziti.mgmt_pb.StreamMetricsEvent.FloatMetricsEntry
+	(*StreamMetricsEvent_IntervalMetric)(nil), // 31: ziti.mgmt_pb.StreamMetricsEvent.IntervalMetric
+	nil,                                  // 32: ziti.mgmt_pb.StreamMetricsEvent.MetricGroupEntry
+	nil,                                  // 33: ziti.mgmt_pb.StreamMetricsEvent.IntervalMetric.ValuesEntry
+	(*InspectResponse_InspectValue)(nil), // 34: ziti.mgmt_pb.InspectResponse.InspectValue
+	(*timestamppb.Timestamp)(nil),        // 35: google.protobuf.Timestamp
 }
 var file_mgmt_proto_depIdxs = []int32{
-	23, // 0: ziti.mgmt_pb.StreamMetricsRequest.matchers:type_name -> ziti.mgmt_pb.StreamMetricsRequest.MetricMatcher
-	31, // 1: ziti.mgmt_pb.StreamMetricsEvent.timestamp:type_name -> google.protobuf.Timestamp
-	24, // 2: ziti.mgmt_pb.StreamMetricsEvent.tags:type_name -> ziti.mgmt_pb.StreamMetricsEvent.TagsEntry
-	25, // 3: ziti.mgmt_pb.StreamMetricsEvent.intMetrics:type_name -> ziti.mgmt_pb.StreamMetricsEvent.IntMetricsEntry
-	26, // 4: ziti.mgmt_pb.StreamMetricsEvent.floatMetrics:type_name -> ziti.mgmt_pb.StreamMetricsEvent.FloatMetricsEntry
-	27, // 5: ziti.mgmt_pb.StreamMetricsEvent.intervalMetrics:type_name -> ziti.mgmt_pb.StreamMetricsEvent.IntervalMetric
-	28, // 6: ziti.mgmt_pb.StreamMetricsEvent.metricGroup:type_name -> ziti.mgmt_pb.StreamMetricsEvent.MetricGroupEntry
+	27, // 0: ziti.mgmt_pb.StreamMetricsRequest.matchers:type_name -> ziti.mgmt_pb.StreamMetricsRequest.MetricMatcher
+	35, // 1: ziti.mgmt_pb.StreamMetricsEvent.timestamp:type_name -> google.protobuf.Timestamp
+	28, // 2: ziti.mgmt_pb.StreamMetricsEvent.tags:type_name -> ziti.mgmt_pb.StreamMetricsEvent.TagsEntry
+	29, // 3: ziti.mgmt_pb.StreamMetricsEvent.intMetrics:type_name -> ziti.mgmt_pb.StreamMetricsEvent.IntMetricsEntry
+	30, // 4: ziti.mgmt_pb.StreamMetricsEvent.floatMetrics:type_name -> ziti.mgmt_pb.StreamMetricsEvent.FloatMetricsEntry
+	31, // 5: ziti.mgmt_pb.StreamMetricsEvent.intervalMetrics:type_name -> ziti.mgmt_pb.StreamMetricsEvent.IntervalMetric
+	32, // 6: ziti.mgmt_pb.StreamMetricsEvent.metricGroup:type_name -> ziti.mgmt_pb.StreamMetricsEvent.MetricGroupEntry
 	2,  // 7: ziti.mgmt_pb.StreamCircuitsEvent.eventType:type_name -> ziti.mgmt_pb.StreamCircuitEventType
 	8,  // 8: ziti.mgmt_pb.StreamCircuitsEvent.path:type_name -> ziti.mgmt_pb.Path
 	3,  // 9: ziti.mgmt_pb.StreamTracesRequest.filterType:type_name -> ziti.mgmt_pb.TraceFilterType
-	30, // 10: ziti.mgmt_pb.InspectResponse.values:type_name -> ziti.mgmt_pb.InspectResponse.InspectValue
+	34, // 10: ziti.mgmt_pb.InspectResponse.values:type_name -> ziti.mgmt_pb.InspectResponse.InspectValue
 	14, // 11: ziti.mgmt_pb.RaftMemberListResponse.members:type_name -> ziti.mgmt_pb.RaftMember
 	4,  // 12: ziti.mgmt_pb.TerminatorDetail.state:type_name -> ziti.mgmt_pb.TerminatorState
 	22, // 13: ziti.mgmt_pb.RouterLinkDetails.linkDetails:type_name -> ziti.mgmt_pb.RouterLinkDetail
 	5,  // 14: ziti.mgmt_pb.RouterLinkDetail.ctrlState:type_name -> ziti.mgmt_pb.LinkState
 	5,  // 15: ziti.mgmt_pb.RouterLinkDetail.routerState:type_name -> ziti.mgmt_pb.LinkState
-	31, // 16: ziti.mgmt_pb.StreamMetricsEvent.IntervalMetric.intervalStartUTC:type_name -> google.protobuf.Timestamp
-	31, // 17: ziti.mgmt_pb.StreamMetricsEvent.IntervalMetric.intervalEndUTC:type_name -> google.protobuf.Timestamp
-	29, // 18: ziti.mgmt_pb.StreamMetricsEvent.IntervalMetric.values:type_name -> ziti.mgmt_pb.StreamMetricsEvent.IntervalMetric.ValuesEntry
-	19, // [19:19] is the sub-list for method output_type
-	19, // [19:19] is the sub-list for method input_type
-	19, // [19:19] is the sub-list for extension type_name
-	19, // [19:19] is the sub-list for extension extendee
-	0,  // [0:19] is the sub-list for field type_name
+	26, // 16: ziti.mgmt_pb.RouterSdkTerminatorsDetails.details:type_name -> ziti.mgmt_pb.RouterSdkTerminatorDetail
+	4,  // 17: ziti.mgmt_pb.RouterSdkTerminatorDetail.ctrlState:type_name -> ziti.mgmt_pb.TerminatorState
+	35, // 18: ziti.mgmt_pb.StreamMetricsEvent.IntervalMetric.intervalStartUTC:type_name -> google.protobuf.Timestamp
+	35, // 19: ziti.mgmt_pb.StreamMetricsEvent.IntervalMetric.intervalEndUTC:type_name -> google.protobuf.Timestamp
+	33, // 20: ziti.mgmt_pb.StreamMetricsEvent.IntervalMetric.values:type_name -> ziti.mgmt_pb.StreamMetricsEvent.IntervalMetric.ValuesEntry
+	21, // [21:21] is the sub-list for method output_type
+	21, // [21:21] is the sub-list for method input_type
+	21, // [21:21] is the sub-list for extension type_name
+	21, // [21:21] is the sub-list for extension extendee
+	0,  // [0:21] is the sub-list for field type_name
 }
 
 func init() { file_mgmt_proto_init() }
@@ -2457,7 +2809,43 @@ func file_mgmt_proto_init() {
 			}
 		}
 		file_mgmt_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*StreamMetricsRequest_MetricMatcher); i {
+			switch v := v.(*ValidateRouterSdkTerminatorsRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_mgmt_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ValidateRouterSdkTerminatorsResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_mgmt_proto_msgTypes[19].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*RouterSdkTerminatorsDetails); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_mgmt_proto_msgTypes[20].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*RouterSdkTerminatorDetail); i {
 			case 0:
 				return &v.state
 			case 1:
@@ -2469,6 +2857,18 @@ func file_mgmt_proto_init() {
 			}
 		}
 		file_mgmt_proto_msgTypes[21].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*StreamMetricsRequest_MetricMatcher); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_mgmt_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} {
 			switch v := v.(*StreamMetricsEvent_IntervalMetric); i {
 			case 0:
 				return &v.state
@@ -2480,7 +2880,7 @@ func file_mgmt_proto_init() {
 				return nil
 			}
 		}
-		file_mgmt_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} {
+		file_mgmt_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} {
 			switch v := v.(*InspectResponse_InspectValue); i {
 			case 0:
 				return &v.state
@@ -2500,7 +2900,7 @@ func file_mgmt_proto_init() {
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
 			RawDescriptor: file_mgmt_proto_rawDesc,
 			NumEnums:      6,
-			NumMessages:   25,
+			NumMessages:   29,
 			NumExtensions: 0,
 			NumServices:   0,
 		},
diff --git a/common/pb/mgmt_pb/mgmt.proto b/common/pb/mgmt_pb/mgmt.proto
index 7290097a1..383d186e8 100644
--- a/common/pb/mgmt_pb/mgmt.proto
+++ b/common/pb/mgmt_pb/mgmt.proto
@@ -54,6 +54,10 @@ enum ContentType {
   ValidateRouterLinksResponseType = 10104;
   ValidateRouterLinksResultType = 10105;
 
+  ValidateRouterSdkTerminatorsRequestType = 10106;
+  ValidateRouterSdkTerminatorsResponseType = 10107;
+  ValidateRouterSdkTerminatorsResultType = 10108;
+
 }
 
 enum Header {
@@ -237,3 +241,31 @@ message RouterLinkDetail {
   string destRouterId = 6;
   bool dialed = 7;
 }
+
+message ValidateRouterSdkTerminatorsRequest {
+  string filter = 1;
+}
+
+message ValidateRouterSdkTerminatorsResponse {
+  bool success = 1;
+  string message = 2;
+  uint64 routerCount = 3;
+}
+
+message RouterSdkTerminatorsDetails {
+  string routerId = 1;
+  string routerName = 2;
+  bool validateSuccess = 3;
+  string message = 4;
+  repeated RouterSdkTerminatorDetail details = 5;
+}
+
+message RouterSdkTerminatorDetail {
+  string terminatorId = 1;
+  TerminatorState ctrlState = 2;
+  string routerState = 3;
+  bool isValid = 4;
+  bool operaationActive = 5;
+  string createTime = 6;
+  string lastAttempt = 7;
+}
\ No newline at end of file
diff --git a/controller/api_impl/circuit_api_model.go b/controller/api_impl/circuit_api_model.go
index aa4428f5e..a0400eb0f 100644
--- a/controller/api_impl/circuit_api_model.go
+++ b/controller/api_impl/circuit_api_model.go
@@ -53,17 +53,26 @@ func MapCircuitToRestModel(n *network.Network, _ api.RequestContext, circuit *ne
 		path.Links = append(path.Links, ToEntityRef(link.Id, link, LinkLinkFactory))
 	}
 
-	svc, err := n.Services.Read(circuit.ServiceId)
-	if err != nil {
-		return nil, err
+	var svcEntityRef *rest_model.EntityRef
+	if svc, _ := n.Services.Read(circuit.ServiceId); svc != nil {
+		svcEntityRef = ToEntityRef(svc.Name, svc, ServiceLinkFactory)
+	} else {
+		svcEntityRef = ToEntityRef("<deleted>", deletedEntity(circuit.ServiceId), ServiceLinkFactory)
 	}
+
 	ret := &rest_model.CircuitDetail{
 		BaseEntity: BaseEntityToRestModel(circuit, CircuitLinkFactory),
 		ClientID:   circuit.ClientId,
 		Path:       path,
-		Service:    ToEntityRef(svc.Name, svc, ServiceLinkFactory),
+		Service:    svcEntityRef,
 		Terminator: ToEntityRef(circuit.Terminator.GetId(), circuit.Terminator, TerminatorLinkFactory),
 	}
 
 	return ret, nil
 }
+
+type deletedEntity string
+
+func (self deletedEntity) GetId() string {
+	return string(self)
+}
diff --git a/controller/change/util.go b/controller/change/util.go
new file mode 100644
index 000000000..8b83d8c1a
--- /dev/null
+++ b/controller/change/util.go
@@ -0,0 +1,30 @@
+/*
+	Copyright NetFoundry Inc.
+
+	Licensed under the Apache License, Version 2.0 (the "License");
+	you may not use this file except in compliance with the License.
+	You may obtain a copy of the License at
+
+	https://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+	distributed under the License is distributed on an "AS IS" BASIS,
+	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+	See the License for the specific language governing permissions and
+	limitations under the License.
+*/
+
+package change
+
+import "github.com/openziti/channel/v2"
+
+func NewControlChannelChange(routerId, routerName, method string, ch channel.Channel) *Context {
+	return New().
+		SetChangeAuthorId(routerId).
+		SetChangeAuthorName(routerName).
+		SetChangeAuthorType(AuthorTypeRouter).
+		SetSourceType(SourceTypeControlChannel).
+		SetSourceMethod(method).
+		SetSourceLocal(ch.Underlay().GetLocalAddr().String()).
+		SetSourceRemote(ch.Underlay().GetRemoteAddr().String())
+}
diff --git a/controller/command/command.go b/controller/command/command.go
index c6eb26cb0..d90af365f 100644
--- a/controller/command/command.go
+++ b/controller/command/command.go
@@ -51,6 +51,7 @@ type Dispatcher interface {
 	Dispatch(command Command) error
 	IsLeaderOrLeaderless() bool
 	GetPeers() map[string]channel.Channel
+	GetRateLimiter() RateLimiter
 }
 
 // LocalDispatcher should be used when running a non-clustered system
@@ -67,6 +68,10 @@ func (self *LocalDispatcher) GetPeers() map[string]channel.Channel {
 	return nil
 }
 
+func (self *LocalDispatcher) GetRateLimiter() RateLimiter {
+	return self.Limiter
+}
+
 func (self *LocalDispatcher) Dispatch(command Command) error {
 	defer func() {
 		if p := recover(); p != nil {
diff --git a/controller/command/rate_limiter.go b/controller/command/rate_limiter.go
index 1ce31d186..f62202c78 100644
--- a/controller/command/rate_limiter.go
+++ b/controller/command/rate_limiter.go
@@ -18,10 +18,13 @@ package command
 
 import (
 	"fmt"
+	"github.com/google/uuid"
+	"github.com/michaelquigley/pfxlog"
 	"github.com/openziti/foundation/v2/errorz"
 	"github.com/openziti/metrics"
 	"github.com/openziti/ziti/controller/apierror"
 	"github.com/pkg/errors"
+	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -33,6 +36,11 @@ const (
 
 	DefaultLimiterSize = 100
 	MinLimiterSize     = 10
+
+	DefaultAdaptiveRateLimiterEnabled       = true
+	DefaultAdaptiveRateLimiterMinWindowSize = 5
+	DefaultAdaptiveRateLimiterMaxWindowSize = 250
+	DefaultAdaptiveRateLimiterTimeout       = 30 * time.Second
 )
 
 type RateLimiterConfig struct {
@@ -53,6 +61,7 @@ func NewRateLimiter(config RateLimiterConfig, registry metrics.Registry, closeNo
 		queue:       make(chan *rateLimitedWork, config.QueueSize),
 		closeNotify: closeNotify,
 		workRate:    registry.Timer(MetricCommandLimiterWorkTimer),
+		config:      config,
 	}
 
 	if existing := registry.GetGauge(MetricCommandLimiterCurrentQueuedCount); existing != nil {
@@ -73,6 +82,7 @@ func NewRateLimiter(config RateLimiterConfig, registry metrics.Registry, closeNo
 // an ApiError indicating that the server is too busy
 type RateLimiter interface {
 	RunRateLimited(func() error) error
+	GetQueueFillPct() float64
 }
 
 // An AdaptiveRateLimiter allows running arbitrary, sequential operations with a limiter, so that only N operations
@@ -96,18 +106,46 @@ type AdaptiveRateLimiter interface {
 	RunRateLimited(f func() error) (RateLimitControl, error)
 }
 
+// An AdaptiveRateLimitTracker works similarly to an AdaptiveRateLimiter, except it just manages the rate
+// limiting without actually running the work. Because it doesn't run the work itself, it has to account
+// for the possibility that some work may never report as complete or failed. It thus has a configurable
+// timeout at which point outstanding work will be marked as failed.
+type AdaptiveRateLimitTracker interface {
+	RunRateLimited(label string) (RateLimitControl, error)
+	RunRateLimitedF(label string, f func(control RateLimitControl) error) error
+	IsRateLimited() bool
+}
+
 type NoOpRateLimiter struct{}
 
 func (self NoOpRateLimiter) RunRateLimited(f func() error) error {
 	return f()
 }
 
+func (self NoOpRateLimiter) GetQueueFillPct() float64 {
+	return 0
+}
+
 type NoOpAdaptiveRateLimiter struct{}
 
 func (self NoOpAdaptiveRateLimiter) RunRateLimited(f func() error) (RateLimitControl, error) {
 	return noOpRateLimitControl{}, f()
 }
 
+type NoOpAdaptiveRateLimitTracker struct{}
+
+func (n NoOpAdaptiveRateLimitTracker) RunRateLimited(string) (RateLimitControl, error) {
+	return noOpRateLimitControl{}, nil
+}
+
+func (n NoOpAdaptiveRateLimitTracker) RunRateLimitedF(_ string, f func(control RateLimitControl) error) error {
+	return f(noOpRateLimitControl{})
+}
+
+func (n NoOpAdaptiveRateLimitTracker) IsRateLimited() bool {
+	return false
+}
+
 type rateLimitedWork struct {
 	wrapped func() error
 	result  chan error
@@ -118,6 +156,7 @@ type DefaultRateLimiter struct {
 	queue       chan *rateLimitedWork
 	closeNotify <-chan struct{}
 	workRate    metrics.Timer
+	config      RateLimiterConfig
 }
 
 func (self *DefaultRateLimiter) RunRateLimited(f func() error) error {
@@ -141,6 +180,10 @@ func (self *DefaultRateLimiter) RunRateLimited(f func() error) error {
 	}
 }
 
+func (self *DefaultRateLimiter) GetQueueFillPct() float64 {
+	return float64(self.currentSize.Load()) / float64(self.config.QueueSize)
+}
+
 func (self *DefaultRateLimiter) run() {
 	defer self.workRate.Dispose()
 
@@ -180,19 +223,58 @@ type AdaptiveRateLimiterConfig struct {
 
 	// WindowSizeMetric - the name of the metric show the current window size
 	WindowSizeMetric string
+
+	// Timeout - only used for AdaptiveRateLimitTracker, sets when a piece of outstanding work will be assumed to
+	//           have failed if it hasn't been marked completed yet, so that work slots aren't lost
+	Timeout time.Duration
+}
+
+func (self *AdaptiveRateLimiterConfig) SetDefaults() {
+	self.Enabled = DefaultAdaptiveRateLimiterEnabled
+	self.MinSize = DefaultAdaptiveRateLimiterMinWindowSize
+	self.MaxSize = DefaultAdaptiveRateLimiterMaxWindowSize
+	self.Timeout = DefaultAdaptiveRateLimiterTimeout
 }
 
-func (self *AdaptiveRateLimiterConfig) Validate() error {
-	if !self.Enabled {
-		return nil
+func LoadAdaptiveRateLimiterConfig(cfg *AdaptiveRateLimiterConfig, cfgmap map[interface{}]interface{}) error {
+	if value, found := cfgmap["enabled"]; found {
+		cfg.Enabled = strings.EqualFold("true", fmt.Sprintf("%v", value))
+	}
+
+	if value, found := cfgmap["maxSize"]; found {
+		if intVal, ok := value.(int); ok {
+			v := int64(intVal)
+			cfg.MaxSize = uint32(v)
+		} else {
+			return errors.Errorf("invalid value %d for adaptive rate limiter max size, must be integer value", value)
+		}
+	}
+
+	if value, found := cfgmap["minSize"]; found {
+		if intVal, ok := value.(int); ok {
+			v := int64(intVal)
+			cfg.MinSize = uint32(v)
+		} else {
+			return errors.Errorf("invalid value %d for adaptive rate limiter min size, must be integer value", value)
+		}
+	}
+
+	if cfg.MinSize < 1 {
+		return errors.Errorf("invalid value %d for adaptive rate limiter min size, must be at least", cfg.MinSize)
 	}
 
-	if self.MinSize < 1 {
-		return errors.New("adaptive rate limiter min size is 1")
+	if cfg.MinSize > cfg.MaxSize {
+		return errors.Errorf("invalid values, %d, %d for adaptive rate limiter min size and max size, min must be <= max",
+			cfg.MinSize, cfg.MaxSize)
 	}
-	if self.MinSize > self.MaxSize {
-		return fmt.Errorf("adaptive rate limiter min size must be <- max size. min: %v, max: %v", self.MinSize, self.MaxSize)
+
+	if value, found := cfgmap["timeout"]; found {
+		var err error
+		if cfg.Timeout, err = time.ParseDuration(fmt.Sprintf("%v", value)); err != nil {
+			return fmt.Errorf("invalid value %v for adaptive rate limiter timeout (%w)", value, err)
+		}
 	}
+
 	return nil
 }
 
@@ -202,12 +284,11 @@ func NewAdaptiveRateLimiter(config AdaptiveRateLimiterConfig, registry metrics.R
 	}
 
 	result := &adaptiveRateLimiter{
-		currentWindow: atomic.Int32{},
-		minWindow:     int32(config.MinSize),
-		maxWindow:     int32(config.MaxSize),
-		queue:         make(chan *adaptiveRateLimitedWork, config.MaxSize),
-		closeNotify:   closeNotify,
-		workRate:      registry.Timer(config.WorkTimerMetric),
+		minWindow:   int32(config.MinSize),
+		maxWindow:   int32(config.MaxSize),
+		queue:       make(chan *adaptiveRateLimitedWork, config.MaxSize),
+		closeNotify: closeNotify,
+		workRate:    registry.Timer(config.WorkTimerMetric),
 	}
 
 	if existing := registry.GetGauge(config.QueueSizeMetric); existing != nil {
@@ -267,7 +348,7 @@ func (self *adaptiveRateLimiter) success() {
 	}
 }
 
-func (self *adaptiveRateLimiter) failure(queuePosition int32) {
+func (self *adaptiveRateLimiter) backoff(queuePosition int32) {
 	if self.currentWindow.Load() <= self.minWindow {
 		return
 	}
@@ -351,8 +432,14 @@ func (self *adaptiveRateLimiter) run() {
 }
 
 type RateLimitControl interface {
+	// Success indicates the operation was a success
 	Success()
-	Timeout()
+
+	// Backoff indicates that we need to backoff
+	Backoff()
+
+	// Failed indicates the operation was not a success, but a backoff isn't required
+	Failed()
 }
 
 type rateLimitControl struct {
@@ -364,15 +451,25 @@ func (r rateLimitControl) Success() {
 	r.limiter.success()
 }
 
-func (r rateLimitControl) Timeout() {
-	r.limiter.failure(r.queuePosition)
+func (r rateLimitControl) Backoff() {
+	r.limiter.backoff(r.queuePosition)
+}
+
+func (r rateLimitControl) Failed() {
+	// no-op for this type
+}
+
+func NoOpRateLimitControl() RateLimitControl {
+	return noOpRateLimitControl{}
 }
 
 type noOpRateLimitControl struct{}
 
 func (noOpRateLimitControl) Success() {}
 
-func (noOpRateLimitControl) Timeout() {}
+func (noOpRateLimitControl) Backoff() {}
+
+func (noOpRateLimitControl) Failed() {}
 
 func WasRateLimited(err error) bool {
 	var apiErr *errorz.ApiError
@@ -381,3 +478,207 @@ func WasRateLimited(err error) bool {
 	}
 	return false
 }
+
+func NewAdaptiveRateLimitTracker(config AdaptiveRateLimiterConfig, registry metrics.Registry, closeNotify <-chan struct{}) AdaptiveRateLimitTracker {
+	if !config.Enabled {
+		return NoOpAdaptiveRateLimitTracker{}
+	}
+
+	result := &adaptiveRateLimitTracker{
+		minWindow:       int32(config.MinSize),
+		maxWindow:       int32(config.MaxSize),
+		timeout:         config.Timeout,
+		workRate:        registry.Timer(config.WorkTimerMetric),
+		outstandingWork: map[string]*adaptiveRateLimitTrackerWork{},
+		closeNotify:     closeNotify,
+	}
+
+	if existing := registry.GetGauge(config.QueueSizeMetric); existing != nil {
+		existing.Dispose()
+	}
+
+	registry.FuncGauge(config.QueueSizeMetric, func() int64 {
+		return int64(result.currentSize.Load())
+	})
+
+	if existing := registry.GetGauge(config.WindowSizeMetric); existing != nil {
+		existing.Dispose()
+	}
+
+	registry.FuncGauge(config.WindowSizeMetric, func() int64 {
+		return int64(result.currentWindow.Load())
+	})
+
+	result.currentWindow.Store(int32(config.MaxSize))
+
+	go result.run()
+
+	return result
+}
+
+type adaptiveRateLimitTracker struct {
+	currentWindow  atomic.Int32
+	minWindow      int32
+	maxWindow      int32
+	timeout        time.Duration
+	lock           sync.Mutex
+	successCounter atomic.Uint32
+
+	currentSize     atomic.Int32
+	workRate        metrics.Timer
+	outstandingWork map[string]*adaptiveRateLimitTrackerWork
+	closeNotify     <-chan struct{}
+}
+
+func (self *adaptiveRateLimitTracker) IsRateLimited() bool {
+	return self.currentSize.Load() >= self.currentWindow.Load()
+}
+
+func (self *adaptiveRateLimitTracker) success(work *adaptiveRateLimitTrackerWork) {
+	self.lock.Lock()
+	defer self.lock.Unlock()
+
+	self.currentSize.Add(-1)
+	delete(self.outstandingWork, work.id)
+	self.workRate.UpdateSince(work.createTime)
+	if self.currentWindow.Load() >= self.maxWindow {
+		return
+	}
+
+	if self.successCounter.Add(1)%10 == 0 {
+		if nextVal := self.currentWindow.Add(1); nextVal > self.maxWindow {
+			self.currentWindow.Store(self.maxWindow)
+		}
+	}
+}
+
+func (self *adaptiveRateLimitTracker) backoff(work *adaptiveRateLimitTrackerWork) {
+	self.lock.Lock()
+	defer self.lock.Unlock()
+
+	self.currentSize.Add(-1)
+	delete(self.outstandingWork, work.id)
+
+	if self.currentWindow.Load() <= self.minWindow {
+		return
+	}
+
+	current := self.currentWindow.Load()
+	nextWindow := work.queuePosition - 10
+	if nextWindow < current {
+		if nextWindow < self.minWindow {
+			nextWindow = self.minWindow
+		}
+		self.currentWindow.Store(nextWindow)
+	}
+}
+
+func (self *adaptiveRateLimitTracker) complete(work *adaptiveRateLimitTrackerWork) {
+	self.lock.Lock()
+	defer self.lock.Unlock()
+	self.currentSize.Add(-1)
+	delete(self.outstandingWork, work.id)
+}
+
+func (self *adaptiveRateLimitTracker) RunRateLimited(label string) (RateLimitControl, error) {
+	self.lock.Lock()
+	defer self.lock.Unlock()
+	queuePosition := self.currentSize.Add(1)
+	if queuePosition > self.currentWindow.Load() {
+		self.currentSize.Add(-1)
+		return noOpRateLimitControl{}, apierror.NewTooManyUpdatesError()
+	}
+
+	work := &adaptiveRateLimitTrackerWork{
+		id:            uuid.NewString(),
+		limiter:       self,
+		queuePosition: queuePosition,
+		createTime:    time.Now(),
+		label:         label,
+	}
+
+	self.outstandingWork[work.id] = work
+
+	return work, nil
+}
+
+func (self *adaptiveRateLimitTracker) RunRateLimitedF(label string, f func(control RateLimitControl) error) error {
+	ctrl, err := self.RunRateLimited(label)
+	if err != nil {
+		return err
+	}
+	return f(ctrl)
+}
+
+func (self *adaptiveRateLimitTracker) run() {
+	defer self.workRate.Dispose()
+
+	ticker := time.NewTicker(30 * time.Second)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ticker.C:
+			self.cleanExpired()
+		case <-self.closeNotify:
+			return
+		}
+	}
+}
+
+func (self *adaptiveRateLimitTracker) cleanExpired() {
+	self.lock.Lock()
+
+	var toRemove []*adaptiveRateLimitTrackerWork
+
+	for _, v := range self.outstandingWork {
+		if time.Since(v.createTime) > self.timeout {
+			toRemove = append(toRemove, v)
+		}
+	}
+
+	self.lock.Unlock()
+
+	for _, work := range toRemove {
+		pfxlog.Logger().WithField("label", work.label).
+			WithField("duration", time.Since(work.createTime)).
+			Error("rate limit work expired")
+		work.Backoff()
+	}
+}
+
+type adaptiveRateLimitTrackerWork struct {
+	id            string
+	limiter       *adaptiveRateLimitTracker
+	queuePosition int32
+	createTime    time.Time
+	completed     atomic.Bool
+	label         string
+}
+
+func (self *adaptiveRateLimitTrackerWork) Success() {
+	if self.completed.CompareAndSwap(false, true) {
+		pfxlog.Logger().WithField("label", self.label).
+			WithField("duration", time.Since(self.createTime)).
+			Info("success")
+		self.limiter.success(self)
+	}
+}
+
+func (self *adaptiveRateLimitTrackerWork) Backoff() {
+	if self.completed.CompareAndSwap(false, true) {
+		pfxlog.Logger().WithField("label", self.label).
+			WithField("duration", time.Since(self.createTime)).
+			Info("backoff")
+		self.limiter.backoff(self)
+	}
+}
+
+func (self *adaptiveRateLimitTrackerWork) Failed() {
+	if self.completed.CompareAndSwap(false, true) {
+		pfxlog.Logger().WithField("label", self.label).
+			WithField("duration", time.Since(self.createTime)).
+			Info("failed")
+		self.limiter.complete(self)
+	}
+}
diff --git a/controller/command/rate_limiter_test.go b/controller/command/rate_limiter_test.go
index 31937f71a..c6384b1f3 100644
--- a/controller/command/rate_limiter_test.go
+++ b/controller/command/rate_limiter_test.go
@@ -21,6 +21,7 @@ package command
 import (
 	"errors"
 	"fmt"
+	"github.com/openziti/foundation/v2/concurrenz"
 	"github.com/openziti/foundation/v2/errorz"
 	"github.com/openziti/metrics"
 	"github.com/openziti/sdk-golang/ziti"
@@ -88,7 +89,7 @@ func Test_AdaptiveRateLimiter(t *testing.T) {
 					elapsed := time.Since(start)
 					if elapsed > time.Second*5 {
 						timedOut.Add(1)
-						ctrl.Timeout()
+						ctrl.Backoff()
 					} else {
 						count++
 						completed.Add(1)
@@ -111,6 +112,90 @@ func Test_AdaptiveRateLimiter(t *testing.T) {
 	logStats()
 }
 
+func Test_AdaptiveRateLimiterTracker(t *testing.T) {
+	cfg := AdaptiveRateLimiterConfig{
+		Enabled:          true,
+		MaxSize:          250,
+		MinSize:          5,
+		WorkTimerMetric:  "workTime",
+		QueueSizeMetric:  "queueSize",
+		WindowSizeMetric: "windowSize",
+		Timeout:          time.Second,
+	}
+
+	registry := metrics.NewRegistry("test", nil)
+	closeNotify := make(chan struct{})
+	limiter := NewAdaptiveRateLimitTracker(cfg, registry, closeNotify).(*adaptiveRateLimitTracker)
+
+	var queueFull atomic.Uint32
+	var timedOut atomic.Uint32
+	var completed atomic.Uint32
+
+	countdown := &sync.WaitGroup{}
+
+	logStats := func() {
+		fmt.Printf("queueFulls: %v\n", queueFull.Load())
+		fmt.Printf("timedOut: %v\n", timedOut.Load())
+		fmt.Printf("completed: %v\n", completed.Load())
+		fmt.Printf("queueSize: %v\n", limiter.currentSize.Load())
+		fmt.Printf("windowSize: %v\n", limiter.currentWindow.Load())
+	}
+
+	go func() {
+		for {
+			select {
+			case <-closeNotify:
+				return
+			case <-time.After(time.Second):
+				logStats()
+			}
+		}
+	}()
+
+	sem := concurrenz.NewSemaphore(25)
+
+	for i := 0; i < 300; i++ {
+		countdown.Add(1)
+
+		go func() {
+			defer countdown.Done()
+			count := 0
+			for count < 1000 {
+				// start := time.Now()
+				err := limiter.RunRateLimitedF("", func(control RateLimitControl) error {
+					if sem.TryAcquire() {
+						time.Sleep(25 * time.Millisecond)
+						control.Success()
+						sem.Release()
+						completed.Add(1)
+					} else {
+						time.Sleep(5 * time.Millisecond)
+						control.Backoff()
+						timedOut.Add(1)
+					}
+					return nil
+				})
+
+				if err != nil {
+					apiError := &errorz.ApiError{}
+					if errors.As(err, &apiError) && apiError.Code == apierror.ServerTooManyRequestsCode {
+						queueFull.Add(1)
+						time.Sleep(time.Millisecond)
+					} else {
+						panic(err)
+					}
+				} else {
+					count++
+				}
+			}
+		}()
+	}
+
+	countdown.Wait()
+	close(closeNotify)
+	logStats()
+}
+
 func Test_AuthFlood(t *testing.T) {
 	countdown := &sync.WaitGroup{}
 
diff --git a/controller/config/config.go b/controller/config/config.go
index be1e92368..ed62d6c0a 100644
--- a/controller/config/config.go
+++ b/controller/config/config.go
@@ -401,53 +401,40 @@ func (c *Config) loadEnrollmentSection(edgeConfigMap map[interface{}]interface{}
 	return nil
 }
 
-func (c *Config) loadAuthRateLimiter(cfgmap map[interface{}]interface{}) error {
+func (c *Config) loadAuthRateLimiterConfig(cfgmap map[interface{}]interface{}) error {
+	c.AuthRateLimiter.SetDefaults()
+
 	c.AuthRateLimiter.Enabled = DefaultAuthRateLimiterEnabled
 	c.AuthRateLimiter.MaxSize = DefaultAuthRateLimiterMaxSize
 	c.AuthRateLimiter.MinSize = DefaultAuthRateLimiterMinSize
 
 	if value, found := cfgmap["authRateLimiter"]; found {
 		if submap, ok := value.(map[interface{}]interface{}); ok {
-			if value, found := submap["enabled"]; found {
-				c.AuthRateLimiter.Enabled = strings.EqualFold("true", fmt.Sprintf("%v", value))
+			if err := command.LoadAdaptiveRateLimiterConfig(&c.AuthRateLimiter, submap); err != nil {
+				return err
 			}
-
-			if value, found := submap["maxSize"]; found {
-				if intVal, ok := value.(int); ok {
-					v := int64(intVal)
-					if v < AuthRateLimiterMinSizeValue {
-						return errors.Errorf("invalid value %v for authRateLimiter.maxSize, must be at least %v", value, AuthRateLimiterMinSizeValue)
-					}
-					if v > AuthRateLimiterMaxSizeValue {
-						return errors.Errorf("invalid value %v for authRateLimiter.maxSize, must be at most %v", value, AuthRateLimiterMaxSizeValue)
-					}
-					c.AuthRateLimiter.MaxSize = uint32(v)
-				} else {
-					return errors.Errorf("invalid value %v for authRateLimiter.maxSize, must be integer value", value)
-				}
+			if c.AuthRateLimiter.MaxSize < AuthRateLimiterMinSizeValue {
+				return errors.Errorf("invalid value %v for authRateLimiter.maxSize, must be at least %v",
+					c.AuthRateLimiter.MaxSize, AuthRateLimiterMinSizeValue)
 			}
-
-			if value, found := submap["minSize"]; found {
-				if intVal, ok := value.(int); ok {
-					v := int64(intVal)
-					if v < AuthRateLimiterMinSizeValue {
-						return errors.Errorf("invalid value %v for authRateLimiter.minSize, must be at least %v", value, AuthRateLimiterMinSizeValue)
-					}
-					if v > AuthRateLimiterMaxSizeValue {
-						return errors.Errorf("invalid value %v for authRateLimiter.minSize, must be at most %v", value, AuthRateLimiterMaxSizeValue)
-					}
-					c.AuthRateLimiter.MinSize = uint32(v)
-				} else {
-					return errors.Errorf("invalid value %v for authRateLimiter.minSize, must be integer value", value)
-				}
+			if c.AuthRateLimiter.MaxSize > AuthRateLimiterMaxSizeValue {
+				return errors.Errorf("invalid value %v for authRateLimiter.maxSize, must be at most %v",
+					c.AuthRateLimiter.MaxSize, AuthRateLimiterMaxSizeValue)
 			}
 
-			if c.AuthRateLimiter.MinSize > c.AuthRateLimiter.MaxSize {
-				return errors.Errorf("invalid values, %v, %v for authRateLimiter minSize and maxSize, min must be <= max",
-					c.AuthRateLimiter.MinSize, c.AuthRateLimiter.MaxSize)
+			if c.AuthRateLimiter.MinSize < AuthRateLimiterMinSizeValue {
+				return errors.Errorf("invalid value %v for authRateLimiter.minSize, must be at least %v",
+					c.AuthRateLimiter.MinSize, AuthRateLimiterMinSizeValue)
 			}
+			if c.AuthRateLimiter.MinSize > AuthRateLimiterMaxSizeValue {
+				return errors.Errorf("invalid value %v for authRateLimiter.minSize, must be at most %v",
+					c.AuthRateLimiter.MinSize, AuthRateLimiterMaxSizeValue)
+			}
+		} else {
+			return errors.Errorf("invalid type for authRateLimiter, should be map instead of %T", value)
 		}
 	}
+
 	return nil
 }
 
@@ -484,7 +471,7 @@ func LoadFromMap(configMap map[interface{}]interface{}) (*Config, error) {
 		return nil, err
 	}
 
-	if err = edgeConfig.loadAuthRateLimiter(edgeConfigMap); err != nil {
+	if err = edgeConfig.loadAuthRateLimiterConfig(edgeConfigMap); err != nil {
 		return nil, err
 	}
 
diff --git a/controller/db/api_session_store.go b/controller/db/api_session_store.go
index 1aaad7937..52e6c4b95 100644
--- a/controller/db/api_session_store.go
+++ b/controller/db/api_session_store.go
@@ -23,6 +23,8 @@ import (
 	"github.com/openziti/storage/boltz"
 	"github.com/openziti/ziti/common/eid"
 	"github.com/openziti/ziti/controller/change"
+	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
 	"go.etcd.io/bbolt"
 	"strings"
 	"time"
@@ -149,44 +151,43 @@ func (store *apiSessionStoreImpl) onEventualDelete(db boltz.Db, name string, api
 		}).Error("error querying for session associated to an api session during onEventualDelete")
 	}
 
-	for _, id := range idCollector.ids {
-		changeContext := change.New().SetSourceType("events.emitter").SetChangeAuthorType(change.AuthorTypeController)
-		err = db.Update(changeContext.NewMutateContext(), func(ctx boltz.MutateContext) error {
-			if err := store.stores.session.DeleteById(ctx, id); err != nil {
-				if boltz.IsErrNotFoundErr(err) {
-					return nil
-				}
-				return err
-			}
-			return nil
-		})
-
-		if err != nil {
-			pfxlog.Logger().WithError(err).WithFields(map[string]interface{}{
-				"eventName":    name,
-				"apiSessionId": string(apiSessionId),
-				"sessionId":    id,
-			}).Error("error deleting for session associated to an api session during onEventualDelete")
-		}
+	if store.stores.rateLimiter.GetQueueFillPct() > 0.5 {
+		time.Sleep(time.Second)
 	}
 
+	store.cleanupSessions(db, name, apiSessionId, idCollector.ids)
+}
+
+func (store *apiSessionStoreImpl) cleanupSessions(db boltz.Db, name string, apiSessionId []byte, sessionIds []string) {
+	logger := pfxlog.Logger().WithField("eventName", name).
+		WithField("apiSessionId", string(apiSessionId))
+
 	changeContext := change.New().SetSourceType("events.emitter").SetChangeAuthorType(change.AuthorTypeController)
-	err = db.Update(changeContext.NewMutateContext(), func(ctx boltz.MutateContext) error {
+	err := db.Update(changeContext.NewMutateContext(), func(ctx boltz.MutateContext) error {
+		indexPath := []string{RootBucket, boltz.IndexesBucket, EntityTypeApiSessions, EntityTypeSessions}
 		if bucket := boltz.Path(ctx.Tx(), indexPath...); bucket != nil {
 			if err := bucket.DeleteBucket(apiSessionId); err != nil {
-				if err != bbolt.ErrBucketNotFound {
-					return err
+				if !errors.Is(err, bbolt.ErrBucketNotFound) {
+					logger.WithError(err).
+						Error("error deleting for api session index associated to an api session during onEventualDelete")
+				}
+			}
+		}
+
+		for _, id := range sessionIds {
+			if err := store.stores.session.DeleteById(ctx, id); err != nil {
+				if !boltz.IsErrNotFoundErr(err) {
+					logger.WithError(err).WithField("sessionId", id).
+						Error("error deleting for session associated to an api session during onEventualDelete")
 				}
 			}
 		}
+
 		return nil
 	})
 
 	if err != nil {
-		pfxlog.Logger().WithError(err).WithFields(map[string]interface{}{
-			"eventName":    name,
-			"apiSessionId": string(apiSessionId),
-		}).Error("error deleting for api session index associated to an api session during onEventualDelete")
+		log.WithError(err).Error("error while cleanup after api-session delete")
 	}
 }
 
diff --git a/controller/db/stores.go b/controller/db/stores.go
index 89b67ec31..2ee31eebe 100644
--- a/controller/db/stores.go
+++ b/controller/db/stores.go
@@ -24,6 +24,7 @@ import (
 	"github.com/openziti/storage/ast"
 	"github.com/openziti/storage/boltz"
 	"github.com/openziti/ziti/controller/change"
+	"github.com/openziti/ziti/controller/command"
 	"go.etcd.io/bbolt"
 	"go4.org/sort"
 	"reflect"
@@ -219,6 +220,8 @@ type stores struct {
 	postureCheckType        *postureCheckTypeStoreImpl
 	apiSessionCertificate   *ApiSessionCertificateStoreImpl
 	mfa                     *MfaStoreImpl
+
+	rateLimiter command.RateLimiter
 }
 
 type DbProvider interface {
@@ -231,13 +234,15 @@ func (f DbProviderF) GetDb() boltz.Db {
 	return f()
 }
 
-func InitStores(db boltz.Db) (*Stores, error) {
+func InitStores(db boltz.Db, rateLimiter command.RateLimiter) (*Stores, error) {
 	dbProvider := DbProviderF(func() boltz.Db {
 		return db
 	})
 	errorHolder := &errorz.ErrorHolderImpl{}
 
-	internalStores := &stores{}
+	internalStores := &stores{
+		rateLimiter: rateLimiter,
+	}
 
 	internalStores.eventualEvent = newEventualEventStore(internalStores)
 	internalStores.EventualEventer = NewEventualEventerBbolt(dbProvider, internalStores.eventualEvent, 2*time.Second, 1000)
diff --git a/controller/db/testing.go b/controller/db/testing.go
index 47dbfb1f7..88dfbf6e2 100644
--- a/controller/db/testing.go
+++ b/controller/db/testing.go
@@ -7,6 +7,7 @@ import (
 	"github.com/openziti/storage/boltztest"
 	"github.com/openziti/ziti/common/eid"
 	"github.com/openziti/ziti/controller/change"
+	"github.com/openziti/ziti/controller/command"
 	"github.com/openziti/ziti/controller/xt"
 	"github.com/openziti/ziti/controller/xt_smartrouting"
 	"github.com/pkg/errors"
@@ -40,7 +41,7 @@ func (ctx *TestContext) Init() {
 	ctx.InitDb(Open)
 
 	var err error
-	ctx.stores, err = InitStores(ctx.GetDb())
+	ctx.stores, err = InitStores(ctx.GetDb(), command.NoOpRateLimiter{})
 	ctx.NoError(err)
 
 	ctx.NoError(RunMigrations(ctx.GetDb(), ctx.stores))
diff --git a/controller/events/metrics_mappers.go b/controller/events/metrics_mappers.go
index acb5783a6..94fdf0d2c 100644
--- a/controller/events/metrics_mappers.go
+++ b/controller/events/metrics_mappers.go
@@ -27,9 +27,10 @@ type ctrlChannelMetricsMapper struct{}
 
 func (ctrlChannelMetricsMapper) mapMetrics(_ *metrics_pb.MetricsMessage, event *event.MetricsEvent) {
 	if strings.HasPrefix(event.Metric, "ctrl.") {
-		parts := strings.Split(event.Metric, ":")
-		event.Metric = parts[0]
-		event.SourceEntityId = parts[1]
+		if parts := strings.Split(event.Metric, ":"); len(parts) > 1 {
+			event.Metric = parts[0]
+			event.SourceEntityId = parts[1]
+		}
 	}
 }
 
diff --git a/controller/handler_ctrl/base.go b/controller/handler_ctrl/base.go
index 1ff98b8bf..9614431c8 100644
--- a/controller/handler_ctrl/base.go
+++ b/controller/handler_ctrl/base.go
@@ -28,12 +28,5 @@ type baseHandler struct {
 }
 
 func (self *baseHandler) newChangeContext(ch channel.Channel, method string) *change.Context {
-	return change.New().
-		SetChangeAuthorId(self.router.Id).
-		SetChangeAuthorName(self.router.Name).
-		SetChangeAuthorType(change.AuthorTypeRouter).
-		SetSourceType(change.SourceTypeControlChannel).
-		SetSourceMethod(method).
-		SetSourceLocal(ch.Underlay().GetLocalAddr().String()).
-		SetSourceRemote(ch.Underlay().GetRemoteAddr().String())
+	return change.NewControlChannelChange(self.router.Id, self.router.Name, method, ch)
 }
diff --git a/controller/handler_ctrl/bind.go b/controller/handler_ctrl/bind.go
index af4ec0b95..a71508d22 100644
--- a/controller/handler_ctrl/bind.go
+++ b/controller/handler_ctrl/bind.go
@@ -17,6 +17,7 @@
 package handler_ctrl
 
 import (
+	"github.com/openziti/ziti/common/pb/ctrl_pb"
 	"github.com/sirupsen/logrus"
 	"time"
 
@@ -72,6 +73,10 @@ func (self *bindHandler) BindChannel(binding channel.Binding) error {
 	binding.AddTypedReceiveHandler(newDequiesceRouterHandler(self.router, self.network))
 	binding.AddTypedReceiveHandler(newDecommissionRouterHandler(self.router, self.network))
 	binding.AddTypedReceiveHandler(newPingHandler())
+	binding.AddTypedReceiveHandler(&channel.AsyncFunctionReceiveAdapter{
+		Type:    int32(ctrl_pb.ContentType_ValidateTerminatorsV2ResponseType),
+		Handler: self.network.RouterMessaging.NewValidationResponseHandler(self.network, self.router),
+	})
 	binding.AddPeekHandler(trace.NewChannelPeekHandler(self.network.GetAppId(), binding.GetChannel(), self.network.GetTraceController()))
 	binding.AddPeekHandler(metrics2.NewCtrlChannelPeekHandler(self.router.Id, self.network.GetMetricsRegistry()))
 
diff --git a/controller/handler_ctrl/remove_terminators.go b/controller/handler_ctrl/remove_terminators.go
index 939025eb9..18689b3c4 100644
--- a/controller/handler_ctrl/remove_terminators.go
+++ b/controller/handler_ctrl/remove_terminators.go
@@ -19,9 +19,10 @@ package handler_ctrl
 import (
 	"github.com/michaelquigley/pfxlog"
 	"github.com/openziti/channel/v2"
-	"github.com/openziti/ziti/controller/network"
 	"github.com/openziti/ziti/common/handler_common"
 	"github.com/openziti/ziti/common/pb/ctrl_pb"
+	"github.com/openziti/ziti/controller/command"
+	"github.com/openziti/ziti/controller/network"
 	"google.golang.org/protobuf/proto"
 )
 
@@ -63,6 +64,8 @@ func (self *removeTerminatorsHandler) handleRemoveTerminators(msg *channel.Messa
 			WithField("terminatorIds", request.TerminatorIds).
 			Info("removed terminators")
 		handler_common.SendSuccess(msg, ch, "")
+	} else if command.WasRateLimited(err) {
+		handler_common.SendServerBusy(msg, ch, "remove.terminators")
 	} else {
 		handler_common.SendFailure(msg, ch, err.Error())
 	}
diff --git a/controller/handler_edge_ctrl/create_terminator_v2.go b/controller/handler_edge_ctrl/create_terminator_v2.go
index 26e8a9d82..cd94c09d5 100644
--- a/controller/handler_edge_ctrl/create_terminator_v2.go
+++ b/controller/handler_edge_ctrl/create_terminator_v2.go
@@ -34,6 +34,7 @@ import (
 	"github.com/sirupsen/logrus"
 	"google.golang.org/protobuf/proto"
 	"math"
+	"time"
 )
 
 type createTerminatorV2Handler struct {
@@ -73,6 +74,7 @@ func (self *createTerminatorV2Handler) HandleReceive(msg *channel.Message, ch ch
 }
 
 func (self *createTerminatorV2Handler) CreateTerminatorV2(ctx *CreateTerminatorV2RequestContext) {
+	start := time.Now()
 	logger := pfxlog.ContextLogger(self.ch.Label()).
 		WithField("routerId", self.ch.Id()).
 		WithField("token", ctx.req.SessionToken).
@@ -89,7 +91,11 @@ func (self *createTerminatorV2Handler) CreateTerminatorV2(ctx *CreateTerminatorV
 	ctx.loadService()
 
 	if ctx.err != nil {
-		self.returnError(ctx, edge_ctrl_pb.CreateTerminatorResult_FailedOther, ctx.err, logger)
+		errCode := edge_ctrl_pb.CreateTerminatorResult_FailedOther
+		if errors.Is(ctx.err, InvalidSessionError{}) {
+			errCode = edge_ctrl_pb.CreateTerminatorResult_FailedInvalidSession
+		}
+		self.returnError(ctx, errCode, ctx.err, logger)
 		return
 	}
 
@@ -146,6 +152,7 @@ func (self *createTerminatorV2Handler) CreateTerminatorV2(ctx *CreateTerminatorV
 			Context: ctx.newChangeContext(),
 		}
 
+		createStart := time.Now()
 		if err := self.appEnv.GetHostController().GetNetwork().Managers.Command.Dispatch(cmd); err != nil {
 			// terminator might have been created while we were trying to create.
 			if terminator, _ = self.getNetwork().Terminators.Read(ctx.req.Address); terminator != nil {
@@ -162,7 +169,9 @@ func (self *createTerminatorV2Handler) CreateTerminatorV2(ctx *CreateTerminatorV
 				return
 			}
 		} else {
-			logger.WithField("terminator", terminator.Id).Info("created terminator")
+			logger.WithField("terminator", terminator.Id).
+				WithField("createTime", time.Since(createStart)).
+				Info("created terminator")
 		}
 	}
 
@@ -183,7 +192,7 @@ func (self *createTerminatorV2Handler) CreateTerminatorV2(ctx *CreateTerminatorV
 		logger.WithError(err).Error("failed to send CreateTunnelTerminatorResponse")
 	}
 
-	logger.Info("completed create terminator v2 operation")
+	logger.WithField("elapsed", time.Since(start)).Info("completed create terminator v2 operation")
 }
 
 func (self *createTerminatorV2Handler) returnError(ctx *CreateTerminatorV2RequestContext, resultType edge_ctrl_pb.CreateTerminatorResult, err error, logger *logrus.Entry) {
diff --git a/controller/handler_edge_ctrl/errors.go b/controller/handler_edge_ctrl/errors.go
index b4fab5b02..818644cc7 100644
--- a/controller/handler_edge_ctrl/errors.go
+++ b/controller/handler_edge_ctrl/errors.go
@@ -1,3 +1,19 @@
+/*
+	Copyright NetFoundry Inc.
+
+	Licensed under the Apache License, Version 2.0 (the "License");
+	you may not use this file except in compliance with the License.
+	You may obtain a copy of the License at
+
+	https://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+	distributed under the License is distributed on an "AS IS" BASIS,
+	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+	See the License for the specific language governing permissions and
+	limitations under the License.
+*/
+
 package handler_edge_ctrl
 
 import "github.com/openziti/sdk-golang/ziti/edge"
diff --git a/controller/handler_edge_ctrl/errors_test.go b/controller/handler_edge_ctrl/errors_test.go
new file mode 100644
index 000000000..3857f7413
--- /dev/null
+++ b/controller/handler_edge_ctrl/errors_test.go
@@ -0,0 +1,29 @@
+/*
+	Copyright NetFoundry Inc.
+
+	Licensed under the Apache License, Version 2.0 (the "License");
+	you may not use this file except in compliance with the License.
+	You may obtain a copy of the License at
+
+	https://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+	distributed under the License is distributed on an "AS IS" BASIS,
+	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+	See the License for the specific language governing permissions and
+	limitations under the License.
+*/
+
+package handler_edge_ctrl
+
+import (
+	"github.com/pkg/errors"
+	"github.com/stretchr/testify/require"
+	"testing"
+)
+
+func Test_ErrorsIs(t *testing.T) {
+	err := error(InvalidSessionError{})
+	req := require.New(t)
+	req.True(errors.Is(err, InvalidSessionError{}))
+}
diff --git a/controller/handler_mgmt/bind.go b/controller/handler_mgmt/bind.go
index a8a1b9539..2c9837ee4 100644
--- a/controller/handler_mgmt/bind.go
+++ b/controller/handler_mgmt/bind.go
@@ -51,6 +51,12 @@ func (bindHandler *BindHandler) BindChannel(binding channel.Binding) error {
 		Handler: validateLinksRequestHandler.HandleReceive,
 	})
 
+	validateSdkTerminatorsRequestHandler := newValidateRouterSdkTerminatorsHandler(bindHandler.network)
+	binding.AddTypedReceiveHandler(&channel.AsyncFunctionReceiveAdapter{
+		Type:    validateSdkTerminatorsRequestHandler.ContentType(),
+		Handler: validateSdkTerminatorsRequestHandler.HandleReceive,
+	})
+
 	tracesHandler := newStreamTracesHandler(bindHandler.network)
 	binding.AddTypedReceiveHandler(tracesHandler)
 	binding.AddCloseHandler(tracesHandler)
diff --git a/controller/handler_mgmt/validate_router_sdk_terminators.go b/controller/handler_mgmt/validate_router_sdk_terminators.go
new file mode 100644
index 000000000..4f8a02a9e
--- /dev/null
+++ b/controller/handler_mgmt/validate_router_sdk_terminators.go
@@ -0,0 +1,88 @@
+/*
+	Copyright NetFoundry Inc.
+
+	Licensed under the Apache License, Version 2.0 (the "License");
+	you may not use this file except in compliance with the License.
+	You may obtain a copy of the License at
+
+	https://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+	distributed under the License is distributed on an "AS IS" BASIS,
+	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+	See the License for the specific language governing permissions and
+	limitations under the License.
+*/
+
+package handler_mgmt
+
+import (
+	"fmt"
+	"github.com/michaelquigley/pfxlog"
+	"github.com/openziti/channel/v2"
+	"github.com/openziti/channel/v2/protobufs"
+	"github.com/openziti/ziti/common/pb/mgmt_pb"
+	"github.com/openziti/ziti/controller/network"
+	"google.golang.org/protobuf/proto"
+	"time"
+)
+
+type validateRouterSdkTerminatorsHandler struct {
+	network *network.Network
+}
+
+func newValidateRouterSdkTerminatorsHandler(network *network.Network) *validateRouterSdkTerminatorsHandler {
+	return &validateRouterSdkTerminatorsHandler{network: network}
+}
+
+func (*validateRouterSdkTerminatorsHandler) ContentType() int32 {
+	return int32(mgmt_pb.ContentType_ValidateRouterSdkTerminatorsRequestType)
+}
+
+func (handler *validateRouterSdkTerminatorsHandler) HandleReceive(msg *channel.Message, ch channel.Channel) {
+	log := pfxlog.ContextLogger(ch.Label())
+	request := &mgmt_pb.ValidateRouterSdkTerminatorsRequest{}
+
+	var err error
+
+	var count int64
+	var evalF func()
+	if err = proto.Unmarshal(msg.Body, request); err == nil {
+		count, evalF, err = handler.network.ValidateRouterSdkTerminators(request.Filter, func(detail *mgmt_pb.RouterSdkTerminatorsDetails) {
+			if !ch.IsClosed() {
+				if sendErr := protobufs.MarshalTyped(detail).WithTimeout(15 * time.Second).SendAndWaitForWire(ch); sendErr != nil {
+					log.WithError(sendErr).Error("send of sdk terminators detail failed, closing channel")
+					if closeErr := ch.Close(); closeErr != nil {
+						log.WithError(closeErr).Error("failed to close channel")
+					}
+				}
+			} else {
+				log.Info("channel closed, unable to send sdk terminators detail")
+			}
+		})
+	}
+
+	response := &mgmt_pb.ValidateRouterSdkTerminatorsResponse{
+		Success:     err == nil,
+		RouterCount: uint64(count),
+	}
+	if err != nil {
+		response.Message = fmt.Sprintf("%v: failed to unmarshall request: %v", handler.network.GetAppId(), err)
+	}
+
+	body, err := proto.Marshal(response)
+	if err != nil {
+		pfxlog.Logger().WithError(err).Error("unexpected error serializing ValidateRouterSdkTerminatorsResponse")
+		return
+	}
+
+	responseMsg := channel.NewMessage(int32(mgmt_pb.ContentType_ValidateRouterSdkTerminatorsResponseType), body)
+	responseMsg.ReplyTo(msg)
+	if err = ch.Send(responseMsg); err != nil {
+		pfxlog.Logger().WithError(err).Error("unexpected error sending ValidateRouterSdkTerminatorsResponse")
+	}
+
+	if evalF != nil {
+		evalF()
+	}
+}
diff --git a/controller/internal/routes/authenticate_router.go b/controller/internal/routes/authenticate_router.go
index 3dcdee71a..3ce97c675 100644
--- a/controller/internal/routes/authenticate_router.go
+++ b/controller/internal/routes/authenticate_router.go
@@ -28,6 +28,7 @@ import (
 	"github.com/openziti/foundation/v2/errorz"
 	"github.com/openziti/metrics"
 	"github.com/openziti/ziti/controller/apierror"
+	"github.com/openziti/ziti/controller/command"
 	"github.com/openziti/ziti/controller/env"
 	"github.com/openziti/ziti/controller/internal/permissions"
 	"github.com/openziti/ziti/controller/model"
@@ -43,7 +44,8 @@ func init() {
 }
 
 type AuthRouter struct {
-	createTimer metrics.Timer
+	createTimer   metrics.Timer
+	lastAdminAuth concurrenz.AtomicValue[time.Time]
 }
 
 func NewAuthRouter() *AuthRouter {
@@ -181,11 +183,23 @@ func (ro *AuthRouter) authHandler(ae *env.AppEnv, rc *response.RequestContext, h
 
 	var sessionIdHolder concurrenz.AtomicValue[string]
 
-	ctrl, err := ae.AuthRateLimiter.RunRateLimited(func() error {
-		sessionId, err := ae.Managers.ApiSession.Create(changeCtx.NewMutateContext(), newApiSession, sessionCerts)
+	lastAdminAuth := ro.lastAdminAuth.Load()
+	allowAdminBypass := identity.IsAdmin && time.Since(lastAdminAuth) > 10*time.Second &&
+		ro.lastAdminAuth.CompareAndSwap(lastAdminAuth, time.Now())
+
+	var ctrl command.RateLimitControl
+	if allowAdminBypass {
+		var sessionId string
+		sessionId, err = ae.Managers.ApiSession.Create(changeCtx.NewMutateContext(), newApiSession, sessionCerts)
 		sessionIdHolder.Store(sessionId)
-		return err
-	})
+		ctrl = command.NoOpRateLimitControl()
+	} else {
+		ctrl, err = ae.AuthRateLimiter.RunRateLimited(func() error {
+			sessionId, err := ae.Managers.ApiSession.Create(changeCtx.NewMutateContext(), newApiSession, sessionCerts)
+			sessionIdHolder.Store(sessionId)
+			return err
+		})
+	}
 
 	if err != nil {
 		rc.RespondWithError(err)
@@ -223,7 +237,7 @@ func (ro *AuthRouter) authHandler(ae *env.AppEnv, rc *response.RequestContext, h
 	if writeOk {
 		ctrl.Success()
 	} else {
-		ctrl.Timeout()
+		ctrl.Backoff()
 	}
 }
 
diff --git a/controller/internal/routes/identity_router.go b/controller/internal/routes/identity_router.go
index 033c55c28..7fdc82f5d 100644
--- a/controller/internal/routes/identity_router.go
+++ b/controller/internal/routes/identity_router.go
@@ -37,6 +37,7 @@ import (
 	"github.com/openziti/ziti/controller/models"
 	"github.com/openziti/ziti/controller/response"
 	"github.com/sirupsen/logrus"
+	"strings"
 	"time"
 )
 
@@ -109,7 +110,9 @@ func (r *IdentityRouter) Register(ae *env.AppEnv) {
 
 	// service list
 	ae.ManagementApi.IdentityListIdentityServicesHandler = identity.ListIdentityServicesHandlerFunc(func(params identity.ListIdentityServicesParams, _ interface{}) middleware.Responder {
-		return ae.IsAllowed(r.listServices, params.HTTPRequest, params.ID, "", permissions.IsAdmin())
+		return ae.IsAllowed(func(ae *env.AppEnv, rc *response.RequestContext) {
+			r.listServices(ae, rc, params)
+		}, params.HTTPRequest, params.ID, "", permissions.IsAdmin())
 	})
 
 	// service configs crud
@@ -239,8 +242,19 @@ func (r *IdentityRouter) listServicePolicies(ae *env.AppEnv, rc *response.Reques
 	ListAssociationWithHandler[*model.Identity, *model.ServicePolicy](ae, rc, ae.Managers.Identity, ae.Managers.ServicePolicy, MapServicePolicyToRestEntity)
 }
 
-func (r *IdentityRouter) listServices(ae *env.AppEnv, rc *response.RequestContext) {
-	filterTemplate := `not isEmpty(from servicePolicies where anyOf(identities) = "%v")`
+func (r *IdentityRouter) listServices(ae *env.AppEnv, rc *response.RequestContext, params identity.ListIdentityServicesParams) {
+	typeFilter := ""
+	if params.PolicyType != nil {
+		if strings.EqualFold(*params.PolicyType, db.PolicyTypeBind.String()) {
+			typeFilter = fmt.Sprintf(` and type = %d`, db.PolicyTypeBind.Id())
+		}
+
+		if strings.EqualFold(*params.PolicyType, db.PolicyTypeDial.String()) {
+			typeFilter = fmt.Sprintf(` and type = %d`, db.PolicyTypeDial.Id())
+		}
+	}
+
+	filterTemplate := `not isEmpty(from servicePolicies where anyOf(identities) = "%v"` + typeFilter + ")"
 	ListAssociationsWithFilter[*model.ServiceDetail](ae, rc, filterTemplate, ae.Managers.EdgeService.GetDetailLister(), MapServiceToRestEntity)
 }
 
diff --git a/controller/internal/routes/service_router.go b/controller/internal/routes/service_router.go
index 2f63bfb7c..6ba1d2c38 100644
--- a/controller/internal/routes/service_router.go
+++ b/controller/internal/routes/service_router.go
@@ -17,12 +17,14 @@
 package routes
 
 import (
+	"fmt"
 	"github.com/go-openapi/runtime/middleware"
 	"github.com/michaelquigley/pfxlog"
 	clientService "github.com/openziti/edge-api/rest_client_api_server/operations/service"
 	managementService "github.com/openziti/edge-api/rest_management_api_server/operations/service"
 	"github.com/openziti/metrics"
 	"github.com/openziti/storage/boltz"
+	"github.com/openziti/ziti/controller/db"
 	"github.com/openziti/ziti/controller/fields"
 	"github.com/openziti/ziti/controller/model"
 	"github.com/openziti/ziti/controller/models"
@@ -105,7 +107,9 @@ func (r *ServiceRouter) Register(ae *env.AppEnv) {
 	})
 
 	ae.ManagementApi.ServiceListServiceIdentitiesHandler = managementService.ListServiceIdentitiesHandlerFunc(func(params managementService.ListServiceIdentitiesParams, _ interface{}) middleware.Responder {
-		return ae.IsAllowed(r.listIdentities, params.HTTPRequest, params.ID, "", permissions.IsAdmin())
+		return ae.IsAllowed(func(ae *env.AppEnv, rc *response.RequestContext) {
+			r.listIdentities(ae, rc, params)
+		}, params.HTTPRequest, params.ID, "", permissions.IsAdmin())
 	})
 
 	ae.ManagementApi.ServiceListServiceConfigHandler = managementService.ListServiceConfigHandlerFunc(func(params managementService.ListServiceConfigParams, _ interface{}) middleware.Responder {
@@ -292,8 +296,19 @@ func (r *ServiceRouter) listClientTerminators(ae *env.AppEnv, rc *response.Reque
 	ListTerminatorAssociations(ae, rc, ae.Managers.EdgeService, ae.Managers.Terminator, MapClientTerminatorToRestEntity)
 }
 
-func (r *ServiceRouter) listIdentities(ae *env.AppEnv, rc *response.RequestContext) {
-	filterTemplate := `not isEmpty(from servicePolicies where anyOf(services) = "%v")`
+func (r *ServiceRouter) listIdentities(ae *env.AppEnv, rc *response.RequestContext, params managementService.ListServiceIdentitiesParams) {
+	typeFilter := ""
+	if params.PolicyType != nil {
+		if strings.EqualFold(*params.PolicyType, db.PolicyTypeBind.String()) {
+			typeFilter = fmt.Sprintf(` and type = %d`, db.PolicyTypeBind.Id())
+		}
+
+		if strings.EqualFold(*params.PolicyType, db.PolicyTypeDial.String()) {
+			typeFilter = fmt.Sprintf(` and type = %d`, db.PolicyTypeDial.Id())
+		}
+	}
+
+	filterTemplate := `not isEmpty(from servicePolicies where anyOf(services) = "%v"` + typeFilter + ")"
 	ListAssociationsWithFilter[*model.Identity](ae, rc, filterTemplate, ae.Managers.Identity, MapIdentityToRestEntity)
 }
 
diff --git a/controller/model/identity_manager.go b/controller/model/identity_manager.go
index 40a5dc4ad..2064df14c 100644
--- a/controller/model/identity_manager.go
+++ b/controller/model/identity_manager.go
@@ -801,13 +801,13 @@ func (statusMap *identityStatusMap) HasEdgeRouterConnection(identityId string) b
 			WithField("identityId", identityId).
 			WithField("expiresAt", stat.expiresAt).
 			WithField("now", now).
-			Debugf("reporting identity from active ER conn pool: timedout")
+			Tracef("reporting identity from active ER conn pool: timedout")
 		return ret
 	}
 
 	pfxlog.Logger().
 		WithField("identityId", identityId).
-		Debugf("reporting identity from active ER conn pool: not found")
+		Tracef("reporting identity from active ER conn pool: not found")
 	return false
 }
 
diff --git a/controller/network/network.go b/controller/network/network.go
index 44df5c90f..6808bc56d 100644
--- a/controller/network/network.go
+++ b/controller/network/network.go
@@ -27,6 +27,7 @@ import (
 	fabricMetrics "github.com/openziti/ziti/common/metrics"
 	"github.com/openziti/ziti/common/pb/mgmt_pb"
 	"github.com/openziti/ziti/controller/event"
+	"github.com/openziti/ziti/controller/raft"
 	"os"
 	"path/filepath"
 	"runtime/debug"
@@ -110,7 +111,7 @@ type Network struct {
 }
 
 func NewNetwork(config Config) (*Network, error) {
-	stores, err := db.InitStores(config.GetDb())
+	stores, err := db.InitStores(config.GetDb(), config.GetCommandDispatcher().GetRateLimiter())
 	if err != nil {
 		return nil, err
 	}
@@ -345,28 +346,40 @@ func (network *Network) ValidateTerminators(r *Router) {
 		return
 	}
 
-	var terminators []*ctrl_pb.Terminator
+	network.Managers.RouterMessaging.ValidateRouterTerminators(result.Entities)
+}
 
-	for _, terminator := range result.Entities {
-		terminators = append(terminators, &ctrl_pb.Terminator{
-			Id:      terminator.Id,
-			Binding: terminator.Binding,
-			Address: terminator.Address,
-		})
-	}
+type LinkValidationCallback func(detail *mgmt_pb.RouterLinkDetails)
 
-	req := &ctrl_pb.ValidateTerminatorsRequest{
-		Terminators: terminators,
+func (n *Network) ValidateLinks(filter string, cb LinkValidationCallback) (int64, func(), error) {
+	result, err := n.Routers.BaseList(filter)
+	if err != nil {
+		return 0, nil, err
 	}
 
-	if err = protobufs.MarshalTyped(req).Send(r.Control); err != nil {
-		logger.WithError(err).Error("unexpected error sending ValidateTerminatorsRequest")
+	sem := concurrenz.NewSemaphore(10)
+
+	evalF := func() {
+		for _, router := range result.Entities {
+			connectedRouter := n.GetConnectedRouter(router.Id)
+			if connectedRouter != nil {
+				sem.Acquire()
+				go func() {
+					defer sem.Release()
+					n.linkController.ValidateRouterLinks(n, connectedRouter, cb)
+				}()
+			} else {
+				n.linkController.reportRouterLinksError(router, errors.New("router not connected"), cb)
+			}
+		}
 	}
+
+	return int64(len(result.Entities)), evalF, nil
 }
 
-type LinkValidationCallback func(detail *mgmt_pb.RouterLinkDetails)
+type SdkTerminatorValidationCallback func(detail *mgmt_pb.RouterSdkTerminatorsDetails)
 
-func (n *Network) ValidateLinks(filter string, cb LinkValidationCallback) (int64, func(), error) {
+func (n *Network) ValidateRouterSdkTerminators(filter string, cb SdkTerminatorValidationCallback) (int64, func(), error) {
 	result, err := n.Routers.BaseList(filter)
 	if err != nil {
 		return 0, nil, err
@@ -381,10 +394,10 @@ func (n *Network) ValidateLinks(filter string, cb LinkValidationCallback) (int64
 				sem.Acquire()
 				go func() {
 					defer sem.Release()
-					n.linkController.ValidateRouterLinks(n, connectedRouter, cb)
+					n.Routers.ValidateRouterSdkTerminators(connectedRouter, cb)
 				}()
 			} else {
-				n.linkController.reportRouterLinksError(router, errors.New("router not connected"), cb)
+				n.Routers.reportRouterSdkTerminatorsError(router, errors.New("router not connected"), cb)
 			}
 		}
 	}
@@ -1263,6 +1276,30 @@ func (network *Network) Inspect(name string) (*string, error) {
 		val, err := json.Marshal(result)
 		strVal := string(val)
 		return &strVal, err
+	} else if lc == "connected-peers" {
+		if raftController, ok := network.Dispatcher.(*raft.Controller); ok {
+			members, err := raftController.ListMembers()
+			if err != nil {
+				return nil, err
+			}
+			result, err := json.Marshal(members)
+			if err != nil {
+				return nil, fmt.Errorf("failed to marshall cluster member list to json (%w)", err)
+			}
+			resultStr := string(result)
+			return &resultStr, nil
+		}
+	} else if lc == "router-messaging" {
+		routerMessagingState, err := network.Managers.RouterMessaging.Inspect()
+		if err != nil {
+			return nil, err
+		}
+		result, err := json.Marshal(routerMessagingState)
+		if err != nil {
+			return nil, fmt.Errorf("failed to marshall router messaging state to json (%w)", err)
+		}
+		resultStr := string(result)
+		return &resultStr, nil
 	}
 
 	return nil, nil
diff --git a/controller/network/router.go b/controller/network/router.go
index 246d9a8af..b6e9cd62e 100644
--- a/controller/network/router.go
+++ b/controller/network/router.go
@@ -17,16 +17,22 @@
 package network
 
 import (
+	"encoding/json"
+	"fmt"
+	"github.com/openziti/channel/v2/protobufs"
 	"github.com/openziti/foundation/v2/genext"
 	"github.com/openziti/foundation/v2/versions"
+	"github.com/openziti/ziti/common/inspect"
 	"github.com/openziti/ziti/common/pb/cmd_pb"
 	"github.com/openziti/ziti/common/pb/ctrl_pb"
+	"github.com/openziti/ziti/common/pb/mgmt_pb"
 	"github.com/openziti/ziti/controller/change"
 	"github.com/openziti/ziti/controller/command"
 	"github.com/openziti/ziti/controller/fields"
 	"github.com/openziti/ziti/controller/xt"
 	"google.golang.org/protobuf/proto"
 	"reflect"
+	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -460,6 +466,100 @@ func (self *RouterManager) Unmarshall(bytes []byte) (*Router, error) {
 	}, nil
 }
 
+func (self *RouterManager) ValidateRouterSdkTerminators(router *Router, cb SdkTerminatorValidationCallback) {
+	request := &ctrl_pb.InspectRequest{RequestedValues: []string{"sdk-terminators"}}
+	resp := &ctrl_pb.InspectResponse{}
+	respMsg, err := protobufs.MarshalTyped(request).WithTimeout(time.Minute).SendForReply(router.Control)
+	if err = protobufs.TypedResponse(resp).Unmarshall(respMsg, err); err != nil {
+		self.reportRouterSdkTerminatorsError(router, err, cb)
+		return
+	}
+
+	var inspectResult *inspect.SdkTerminatorInspectResult
+	for _, val := range resp.Values {
+		if val.Name == "sdk-terminators" {
+			if err = json.Unmarshal([]byte(val.Value), &inspectResult); err != nil {
+				self.reportRouterSdkTerminatorsError(router, err, cb)
+				return
+			}
+		}
+	}
+
+	if inspectResult == nil {
+		if len(resp.Errors) > 0 {
+			err = errors.New(strings.Join(resp.Errors, ","))
+			self.reportRouterSdkTerminatorsError(router, err, cb)
+			return
+		}
+		self.reportRouterSdkTerminatorsError(router, errors.New("no terminator details returned from router"), cb)
+		return
+	}
+
+	listResult, err := self.Terminators.BaseList(fmt.Sprintf(`router="%s" and binding="edge" limit none`, router.Id))
+	if err != nil {
+		self.reportRouterSdkTerminatorsError(router, err, cb)
+		return
+	}
+
+	result := &mgmt_pb.RouterSdkTerminatorsDetails{
+		RouterId:        router.Id,
+		RouterName:      router.Name,
+		ValidateSuccess: true,
+	}
+
+	terminators := map[string]*Terminator{}
+
+	for _, terminator := range listResult.Entities {
+		terminators[terminator.Id] = terminator
+	}
+
+	for _, entry := range inspectResult.Entries {
+		detail := &mgmt_pb.RouterSdkTerminatorDetail{
+			TerminatorId:     entry.Id,
+			RouterState:      entry.State,
+			IsValid:          true,
+			OperaationActive: entry.OperationActive,
+			CreateTime:       entry.CreateTime,
+			LastAttempt:      entry.LastAttempt,
+		}
+		result.Details = append(result.Details, detail)
+
+		if entry.State != "established" {
+			detail.IsValid = false
+		}
+
+		if _, found := terminators[entry.Id]; found {
+			detail.CtrlState = mgmt_pb.TerminatorState_Valid
+			delete(terminators, entry.Id)
+		} else {
+			detail.CtrlState = mgmt_pb.TerminatorState_Unknown
+			detail.IsValid = false
+		}
+	}
+
+	for _, terminator := range terminators {
+		detail := &mgmt_pb.RouterSdkTerminatorDetail{
+			TerminatorId: terminator.Id,
+			CtrlState:    mgmt_pb.TerminatorState_Valid,
+			RouterState:  "unknown",
+			IsValid:      false,
+		}
+		result.Details = append(result.Details, detail)
+	}
+
+	cb(result)
+}
+
+func (self *RouterManager) reportRouterSdkTerminatorsError(router *Router, err error, cb SdkTerminatorValidationCallback) {
+	result := &mgmt_pb.RouterSdkTerminatorsDetails{
+		RouterId:        router.Id,
+		RouterName:      router.Name,
+		ValidateSuccess: false,
+		Message:         err.Error(),
+	}
+	cb(result)
+}
+
 type RouterLinks struct {
 	sync.Mutex
 	allLinks     atomic.Value
diff --git a/controller/network/router_messaging.go b/controller/network/router_messaging.go
index 31211390e..3cfd75966 100644
--- a/controller/network/router_messaging.go
+++ b/controller/network/router_messaging.go
@@ -17,11 +17,17 @@
 package network
 
 import (
+	"errors"
 	"github.com/michaelquigley/pfxlog"
+	"github.com/openziti/channel/v2"
 	"github.com/openziti/channel/v2/protobufs"
 	"github.com/openziti/foundation/v2/goroutines"
+	"github.com/openziti/storage/boltz"
+	"github.com/openziti/ziti/common/inspect"
 	"github.com/openziti/ziti/common/pb/ctrl_pb"
-	log "github.com/sirupsen/logrus"
+	"github.com/openziti/ziti/controller/change"
+	"github.com/openziti/ziti/controller/db"
+	"github.com/openziti/ziti/controller/xt"
 	"sync/atomic"
 	"time"
 )
@@ -37,24 +43,49 @@ func (self *routerUpdates) stateUpdated(routerId string) {
 	self.changedRouters[routerId] = struct{}{}
 }
 
+type terminatorInfo struct {
+	xt.Terminator
+	marker uint64
+}
+type terminatorValidations struct {
+	terminators     map[string]terminatorInfo
+	checkInProgress atomic.Bool
+	lastSend        time.Time
+}
+
 type routerEvent interface {
 	handle(c *RouterMessaging)
 }
 
 func NewRouterMessaging(managers *Managers, routerCommPool goroutines.Pool) *RouterMessaging {
-	return &RouterMessaging{
-		managers:       managers,
-		eventsC:        make(chan routerEvent, 16),
-		routers:        map[string]*routerUpdates{},
-		routerCommPool: routerCommPool,
+	result := &RouterMessaging{
+		managers:              managers,
+		eventsC:               make(chan routerEvent, 16),
+		routerUpdates:         map[string]*routerUpdates{},
+		terminatorValidations: map[string]*terminatorValidations{},
+		routerCommPool:        routerCommPool,
 	}
+
+	managers.stores.Terminator.AddEntityEventListenerF(result.TerminatorCreated, boltz.EntityCreated)
+
+	return result
 }
 
 type RouterMessaging struct {
-	managers       *Managers
-	eventsC        chan routerEvent
-	routers        map[string]*routerUpdates
-	routerCommPool goroutines.Pool
+	managers              *Managers
+	eventsC               chan routerEvent
+	routerUpdates         map[string]*routerUpdates
+	terminatorValidations map[string]*terminatorValidations
+	routerCommPool        goroutines.Pool
+	markerCounter         atomic.Uint64
+}
+
+func (self *RouterMessaging) getNextMarker() uint64 {
+	result := self.markerCounter.Add(1)
+	for result == 0 {
+		result = self.markerCounter.Add(1)
+	}
+	return result
 }
 
 func (self *RouterMessaging) RouterConnected(r *Router) {
@@ -69,6 +100,12 @@ func (self *RouterMessaging) RouterDeleted(routerId string) {
 	self.routerChanged(routerId, false)
 }
 
+func (self *RouterMessaging) TerminatorCreated(terminator *db.Terminator) {
+	self.queueEvent(&terminatorCreatedEvent{
+		terminator: terminator,
+	})
+}
+
 func (self *RouterMessaging) routerChanged(routerId string, connected bool) {
 	self.queueEvent(&routerChangedEvent{
 		routerId:  routerId,
@@ -106,30 +143,52 @@ func (self *RouterMessaging) run() {
 			}
 		}
 
-		if len(self.routers) > 0 {
+		if len(self.routerUpdates) > 0 {
 			self.syncStates()
 		}
+
+		if len(self.terminatorValidations) > 0 {
+			self.sendTerminatorValidationRequests()
+		}
 	}
 }
 
 func (self *RouterMessaging) getRouterStates(routerId string) *routerUpdates {
-	result, found := self.routers[routerId]
+	result, found := self.routerUpdates[routerId]
 	if !found {
 		result = &routerUpdates{
 			changedRouters: map[string]struct{}{},
 		}
-		self.routers[routerId] = result
+		self.routerUpdates[routerId] = result
+	}
+	return result
+}
+
+func (self *RouterMessaging) getTerminatorValidations(routerId string) *terminatorValidations {
+	result, found := self.terminatorValidations[routerId]
+	if !found {
+		result = &terminatorValidations{
+			terminators: map[string]terminatorInfo{},
+		}
+		self.terminatorValidations[routerId] = result
 	}
 	return result
 }
 
 func (self *RouterMessaging) syncStates() {
-	for k, v := range self.routers {
+	for k, v := range self.routerUpdates {
 		notifyRouterId := k
 		updates := v
 		changes := &ctrl_pb.PeerStateChanges{}
 		notifyRouter := self.managers.Routers.getConnected(notifyRouterId)
 		if notifyRouter == nil {
+			// if the router disconnected, we're going to sync everything anyway, so clear anything pending here
+			delete(self.routerUpdates, k)
+			continue
+		}
+
+		if v.sendInProgress.Load() {
+
 			continue
 		}
 
@@ -165,7 +224,10 @@ func (self *RouterMessaging) syncStates() {
 			}
 		}
 
-		updates.sendInProgress.Store(true)
+		if !updates.sendInProgress.CompareAndSwap(false, true) {
+			continue
+		}
+
 		currentStatesVersion := updates.version
 		queueErr := self.routerCommPool.QueueOrError(func() {
 			ch := notifyRouter.Control
@@ -179,7 +241,7 @@ func (self *RouterMessaging) syncStates() {
 				success = false
 			}
 
-			self.queueEvent(&routerSendDone{
+			self.queueEvent(&routerPeerChangesSendDone{
 				routerId: notifyRouter.Id,
 				version:  currentStatesVersion,
 				success:  success,
@@ -193,13 +255,129 @@ func (self *RouterMessaging) syncStates() {
 	}
 }
 
+func (self *RouterMessaging) sendTerminatorValidationRequests() {
+	for routerId, updates := range self.terminatorValidations {
+		self.sendTerminatorValidationRequest(routerId, updates)
+	}
+}
+
+func (self *RouterMessaging) sendTerminatorValidationRequest(routerId string, updates *terminatorValidations) {
+	notifyRouter := self.managers.Routers.getConnected(routerId)
+	if notifyRouter == nil {
+		// if the router disconnected, we're going to sync everything anyway, so clear anything pending here
+		delete(self.terminatorValidations, routerId)
+		return
+	}
+
+	if updates.checkInProgress.Load() {
+		if time.Since(updates.lastSend) > 3*time.Minute {
+			updates.checkInProgress.Store(false)
+		} else {
+			return
+		}
+	}
+
+	var terminators []*ctrl_pb.Terminator
+
+	for _, terminator := range updates.terminators {
+		if time.Since(terminator.GetCreatedAt()) > 5*time.Second {
+			pfxlog.Logger().WithField("terminatorId", terminator.GetId()).Info("queuing validate of terminator")
+			terminators = append(terminators, &ctrl_pb.Terminator{
+				Id:      terminator.GetId(),
+				Binding: terminator.GetBinding(),
+				Address: terminator.GetAddress(),
+				Marker:  terminator.marker,
+			})
+		}
+	}
+
+	if len(terminators) == 0 || !updates.checkInProgress.CompareAndSwap(false, true) {
+		return
+	}
+
+	req := &ctrl_pb.ValidateTerminatorsV2Request{
+		Terminators: terminators,
+		FixInvalid:  true,
+	}
+
+	queueErr := self.routerCommPool.QueueOrError(func() {
+		ch := notifyRouter.Control
+		if ch == nil {
+			return
+		}
+
+		if self.managers.Dispatcher.IsLeaderOrLeaderless() {
+			if err := protobufs.MarshalTyped(req).WithTimeout(time.Second * 1).SendAndWaitForWire(ch); err != nil {
+				pfxlog.Logger().WithError(err).WithField("routerId", notifyRouter.Id).Error("failed to send validate terminators request to router")
+			}
+		}
+	})
+
+	if queueErr != nil {
+		updates.checkInProgress.Store(false)
+	} else {
+		updates.lastSend = time.Now()
+	}
+}
+
+func (self *RouterMessaging) NewValidationResponseHandler(n *Network, r *Router) channel.ReceiveHandlerF {
+	return func(m *channel.Message, ch channel.Channel) {
+		log := pfxlog.Logger().WithField("routerId", r.Id)
+		resp := &ctrl_pb.ValidateTerminatorsV2Response{}
+		if err := protobufs.TypedResponse(resp).Unmarshall(m, nil); err != nil {
+			log.WithError(err).Error("unable to unmarshall validate terminators v2 response")
+			return
+		}
+
+		changeCtx := change.NewControlChannelChange(r.Id, r.Name, "fabric.validate.terminator", ch)
+
+		handler := &terminatorValidationRespReceived{
+			router:    r,
+			changeCtx: changeCtx,
+			resp:      resp,
+		}
+		handler.DeleteInvalid(n)
+		self.queueEvent(handler)
+	}
+}
+
+func (self *RouterMessaging) ValidateRouterTerminators(terminators []*Terminator) {
+	self.queueEvent(&validateTerminators{
+		terminators: terminators,
+	})
+}
+
+func (self *RouterMessaging) Inspect() (*inspect.RouterMessagingState, error) {
+	evt := &routerMessagingInspectEvent{
+		resultC: make(chan *inspect.RouterMessagingState, 1),
+	}
+
+	timeout := time.After(time.Second)
+
+	select {
+	case self.eventsC <- evt:
+	case <-timeout:
+		return nil, errors.New("timed out waiting to queue inspect event to router messaging")
+	}
+
+	select {
+	case result := <-evt.resultC:
+		return result, nil
+	case <-timeout:
+		return nil, errors.New("timed out waiting for inspect result from router messaging")
+	}
+}
+
 type routerChangedEvent struct {
 	routerId  string
 	connected bool
 }
 
 func (self *routerChangedEvent) handle(c *RouterMessaging) {
-	log.Infof("calculating router updates for router %v, connected=%v", self.routerId, self.connected)
+	pfxlog.Logger().WithField("routerId", self.routerId).
+		WithField("connected", self.connected).
+		Info("calculating router updates for router")
+
 	routers := c.managers.Routers.allConnected()
 
 	var sourceRouterState *routerUpdates
@@ -218,18 +396,158 @@ func (self *routerChangedEvent) handle(c *RouterMessaging) {
 	}
 }
 
-type routerSendDone struct {
+type terminatorCreatedEvent struct {
+	terminator *db.Terminator
+}
+
+func (self *terminatorCreatedEvent) handle(c *RouterMessaging) {
+	routerStates := c.getTerminatorValidations(self.terminator.Router)
+	routerStates.terminators[self.terminator.Id] = terminatorInfo{
+		Terminator: self.terminator,
+		marker:     c.getNextMarker(),
+	}
+}
+
+type routerPeerChangesSendDone struct {
 	routerId string
 	version  uint32
 	success  bool
 	states   *routerUpdates
 }
 
-func (self *routerSendDone) handle(c *RouterMessaging) {
-	if states, ok := c.routers[self.routerId]; ok {
+func (self *routerPeerChangesSendDone) handle(c *RouterMessaging) {
+	defer self.states.sendInProgress.Store(false)
+
+	if states, ok := c.routerUpdates[self.routerId]; ok {
 		if self.success && self.version == states.version {
-			delete(c.routers, self.routerId)
+			delete(c.routerUpdates, self.routerId)
+		}
+	}
+}
+
+type validateTerminators struct {
+	terminators []*Terminator
+}
+
+func (self *validateTerminators) handle(c *RouterMessaging) {
+	var currentRouterId string
+	var validations *terminatorValidations
+
+	routers := map[string]*terminatorValidations{}
+
+	for _, terminator := range self.terminators {
+		if terminator.Router != currentRouterId || validations == nil {
+			validations = c.getTerminatorValidations(terminator.Router)
+			currentRouterId = terminator.Router
+			routers[currentRouterId] = validations
+		}
+		validations.terminators[terminator.Id] = terminatorInfo{
+			Terminator: terminator,
+			marker:     c.getNextMarker(),
+		}
+	}
+}
+
+type terminatorValidationRespReceived struct {
+	router    *Router
+	changeCtx *change.Context
+	resp      *ctrl_pb.ValidateTerminatorsV2Response
+	success   bool
+}
+
+func (self *terminatorValidationRespReceived) DeleteInvalid(n *Network) {
+	log := pfxlog.Logger().WithField("routerId", self.router.Id)
+
+	var toDelete []string
+	for terminatorId, state := range self.resp.States {
+		if !state.Valid {
+			toDelete = append(toDelete, terminatorId)
+			log.WithField("terminatorId", terminatorId).
+				WithField("reason", state.Reason.String()).
+				Info("queuing terminator for delete")
 		}
 	}
-	self.states.sendInProgress.Store(false)
+
+	if len(toDelete) > 0 {
+		if err := n.Managers.Terminators.DeleteBatch(toDelete, self.changeCtx); err != nil {
+			for _, terminatorId := range toDelete {
+				log.WithField("terminatorId", terminatorId).
+					WithError(err).
+					Info("batch delete failed")
+			}
+		} else {
+			self.success = true
+		}
+	}
+}
+
+func (self *terminatorValidationRespReceived) handle(c *RouterMessaging) {
+	states := c.getTerminatorValidations(self.router.Id)
+	defer states.checkInProgress.Store(false)
+
+	for terminatorId, state := range self.resp.States {
+		if terminator, ok := states.terminators[terminatorId]; ok {
+			if (state.Valid && (state.Marker == 0 || state.Marker == terminator.marker)) || self.success {
+				delete(states.terminators, terminatorId)
+			}
+		}
+	}
+
+	if len(states.terminators) == 0 {
+		delete(c.terminatorValidations, self.router.Id)
+	}
+}
+
+type routerMessagingInspectEvent struct {
+	resultC chan *inspect.RouterMessagingState
+}
+
+func (self *routerMessagingInspectEvent) handle(c *RouterMessaging) {
+	result := &inspect.RouterMessagingState{}
+
+	getRouterName := func(routerId string) string {
+		if router, _ := c.managers.Routers.Read(routerId); router != nil {
+			return router.Name
+		}
+		return "<unknown>"
+	}
+
+	for routerId, updates := range c.routerUpdates {
+		u := &inspect.RouterUpdates{
+			Router: inspect.RouterInfo{
+				Id:   routerId,
+				Name: getRouterName(routerId),
+			},
+			Version: updates.version,
+		}
+		for updatedRouterId := range updates.changedRouters {
+			u.ChangedRouters = append(u.ChangedRouters, inspect.RouterInfo{
+				Id:   updatedRouterId,
+				Name: getRouterName(updatedRouterId),
+			})
+		}
+		result.RouterUpdates = append(result.RouterUpdates, u)
+	}
+
+	for routerId, pendingValidations := range c.terminatorValidations {
+		v := &inspect.TerminatorValidations{
+			Router: inspect.RouterInfo{
+				Id:   routerId,
+				Name: getRouterName(routerId),
+			},
+			CheckInProgress: pendingValidations.checkInProgress.Load(),
+			LastSend:        pendingValidations.lastSend.Format("2006-01-02 15:04:05"),
+		}
+
+		for terminatorId := range pendingValidations.terminators {
+			v.Terminators = append(v.Terminators, terminatorId)
+		}
+
+		result.TerminatorValidations = append(result.TerminatorValidations, v)
+	}
+
+	select {
+	case self.resultC <- result:
+	default:
+	}
 }
diff --git a/controller/raft/member.go b/controller/raft/member.go
index 532e70049..6ffcb0b1e 100644
--- a/controller/raft/member.go
+++ b/controller/raft/member.go
@@ -28,12 +28,12 @@ import (
 )
 
 type Member struct {
-	Id        string
-	Addr      string
-	Voter     bool
-	Leader    bool
-	Version   string
-	Connected bool
+	Id        string `json:"id"`
+	Addr      string `json:"addr"`
+	Voter     bool   `json:"isVoter"`
+	Leader    bool   `json:"isLeader"`
+	Version   string `json:"version"`
+	Connected bool   `json:"isConnected"`
 }
 
 func (self *Controller) ListMembers() ([]*Member, error) {
diff --git a/controller/raft/raft.go b/controller/raft/raft.go
index 6eb939584..e0d98a961 100644
--- a/controller/raft/raft.go
+++ b/controller/raft/raft.go
@@ -247,6 +247,10 @@ func (self *Controller) GetMesh() mesh.Mesh {
 	return self.Mesh
 }
 
+func (self *Controller) GetRateLimiter() command.RateLimiter {
+	return self.commandRateLimiter
+}
+
 func (self *Controller) ConfigureMeshHandlers(bindHandler channel.BindHandler) {
 	self.Mesh.Init(bindHandler)
 }
diff --git a/go.mod b/go.mod
index e1742866b..4586eedd4 100644
--- a/go.mod
+++ b/go.mod
@@ -22,7 +22,7 @@ require (
 	github.com/go-openapi/loads v0.21.5
 	github.com/go-openapi/runtime v0.27.1
 	github.com/go-openapi/spec v0.20.14
-	github.com/go-openapi/strfmt v0.22.0
+	github.com/go-openapi/strfmt v0.22.1
 	github.com/go-openapi/swag v0.22.9
 	github.com/go-openapi/validate v0.23.0
 	github.com/go-resty/resty/v2 v2.11.0
@@ -35,7 +35,7 @@ require (
 	github.com/gorilla/websocket v1.5.1
 	github.com/hashicorp/go-hclog v1.6.2
 	github.com/hashicorp/golang-lru/v2 v2.0.7
-	github.com/hashicorp/raft v1.6.0
+	github.com/hashicorp/raft v1.6.1
 	github.com/hashicorp/raft-boltdb v0.0.0-20220329195025-15018e9b97e0
 	github.com/jedib0t/go-pretty/v6 v6.5.4
 	github.com/jessevdk/go-flags v1.5.0
@@ -49,7 +49,7 @@ require (
 	github.com/natefinch/lumberjack v2.0.0+incompatible
 	github.com/openziti/agent v1.0.16
 	github.com/openziti/channel/v2 v2.0.119
-	github.com/openziti/edge-api v0.26.11
+	github.com/openziti/edge-api v0.26.12
 	github.com/openziti/foundation/v2 v2.0.37
 	github.com/openziti/identity v1.0.70
 	github.com/openziti/jwks v1.0.3
@@ -77,12 +77,12 @@ require (
 	github.com/teris-io/shortid v0.0.0-20201117134242-e59966efd125
 	github.com/xeipuuv/gojsonschema v1.2.0
 	github.com/zitadel/oidc/v2 v2.12.0
-	go.etcd.io/bbolt v1.3.8
+	go.etcd.io/bbolt v1.3.9
 	go4.org v0.0.0-20180809161055-417644f6feb5
-	golang.org/x/crypto v0.19.0
+	golang.org/x/crypto v0.21.0
 	golang.org/x/net v0.21.0
 	golang.org/x/sync v0.6.0
-	golang.org/x/sys v0.17.0
+	golang.org/x/sys v0.18.0
 	golang.org/x/text v0.14.0
 	google.golang.org/protobuf v1.32.0
 	gopkg.in/AlecAivazis/survey.v1 v1.8.7
@@ -112,7 +112,7 @@ require (
 	github.com/eliukblau/pixterm/pkg/ansimage v0.0.0-20191210081756-9fb6cf8c2f75 // indirect
 	github.com/felixge/httpsnoop v1.0.3 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
-	github.com/go-jose/go-jose/v3 v3.0.1 // indirect
+	github.com/go-jose/go-jose/v3 v3.0.3 // indirect
 	github.com/go-logr/logr v1.4.1 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.3.0 // indirect
@@ -176,16 +176,16 @@ require (
 	github.com/yusufpapurcu/wmi v1.2.3 // indirect
 	go.mongodb.org/mongo-driver v1.14.0 // indirect
 	go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 // indirect
-	go.opentelemetry.io/otel v1.23.1 // indirect
-	go.opentelemetry.io/otel/metric v1.23.1 // indirect
-	go.opentelemetry.io/otel/trace v1.23.1 // indirect
+	go.opentelemetry.io/otel v1.24.0 // indirect
+	go.opentelemetry.io/otel/metric v1.24.0 // indirect
+	go.opentelemetry.io/otel/trace v1.24.0 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.9.0 // indirect
 	golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
 	golang.org/x/image v0.13.0 // indirect
 	golang.org/x/mod v0.14.0 // indirect
 	golang.org/x/oauth2 v0.16.0 // indirect
-	golang.org/x/term v0.17.0 // indirect
+	golang.org/x/term v0.18.0 // indirect
 	golang.org/x/tools v0.17.0 // indirect
 	google.golang.org/appengine v1.6.8 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
diff --git a/go.sum b/go.sum
index cc2e357f8..330007943 100644
--- a/go.sum
+++ b/go.sum
@@ -204,8 +204,8 @@ github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-jose/go-jose/v3 v3.0.1 h1:pWmKFVtt+Jl0vBZTIpz/eAKwsm6LkIxDVVbFHKkchhA=
-github.com/go-jose/go-jose/v3 v3.0.1/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8=
+github.com/go-jose/go-jose/v3 v3.0.3 h1:fFKWeig/irsp7XD2zBxvnmA/XaRWp5V3CBsZXJF7G7k=
+github.com/go-jose/go-jose/v3 v3.0.3/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
@@ -232,8 +232,8 @@ github.com/go-openapi/runtime v0.27.1 h1:ae53yaOoh+fx/X5Eaq8cRmavHgDma65XPZuvBqv
 github.com/go-openapi/runtime v0.27.1/go.mod h1:fijeJEiEclyS8BRurYE1DE5TLb9/KZl6eAdbzjsrlLU=
 github.com/go-openapi/spec v0.20.14 h1:7CBlRnw+mtjFGlPDRZmAMnq35cRzI91xj03HVyUi/Do=
 github.com/go-openapi/spec v0.20.14/go.mod h1:8EOhTpBoFiask8rrgwbLC3zmJfz4zsCUueRuPM6GNkw=
-github.com/go-openapi/strfmt v0.22.0 h1:Ew9PnEYc246TwrEspvBdDHS4BVKXy/AOVsfqGDgAcaI=
-github.com/go-openapi/strfmt v0.22.0/go.mod h1:HzJ9kokGIju3/K6ap8jL+OlGAbjpSv27135Yr9OivU4=
+github.com/go-openapi/strfmt v0.22.1 h1:5Ky8cybT4576C6Ffc+8gYji/wRXCo6Ozm8RaWjPI6jc=
+github.com/go-openapi/strfmt v0.22.1/go.mod h1:OfVoytIXJasDkkGvkb1Cceb3BPyMOwk1FgmyyEw7NYg=
 github.com/go-openapi/swag v0.22.9 h1:XX2DssF+mQKM2DHsbgZK74y/zj4mo9I99+89xUmuZCE=
 github.com/go-openapi/swag v0.22.9/go.mod h1:3/OXnFfnMAwBD099SwYRk7GD3xOrr1iL7d/XNLXVVwE=
 github.com/go-openapi/validate v0.23.0 h1:2l7PJLzCis4YUGEoW6eoQw3WhyM65WSIcjX6SQnlfDw=
@@ -386,8 +386,8 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO
 github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
 github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
 github.com/hashicorp/raft v1.1.0/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM=
-github.com/hashicorp/raft v1.6.0 h1:tkIAORZy2GbJ2Trp5eUSggLXDPOJLXC+JJLNMMqtgtM=
-github.com/hashicorp/raft v1.6.0/go.mod h1:Xil5pDgeGwRWuX4uPUmwa+7Vagg4N804dz6mhNi6S7o=
+github.com/hashicorp/raft v1.6.1 h1:v/jm5fcYHvVkL0akByAp+IDdDSzCNCGhdO6VdB56HIM=
+github.com/hashicorp/raft v1.6.1/go.mod h1:N1sKh6Vn47mrWvEArQgILTyng8GoDRNYlgKyK7PMjs0=
 github.com/hashicorp/raft-boltdb v0.0.0-20220329195025-15018e9b97e0 h1:CO8dBMLH6dvE1jTn/30ZZw3iuPsNfajshWoJTnVc5cc=
 github.com/hashicorp/raft-boltdb v0.0.0-20220329195025-15018e9b97e0/go.mod h1:nTakvJ4XYq45UXtn0DbwR4aU9ZdjlnIenpbs6Cd+FM0=
 github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
@@ -568,8 +568,8 @@ github.com/openziti/channel/v2 v2.0.119 h1:stfSrnDqoTi78LMvQA3+NSivHjQnRrYKrgij5
 github.com/openziti/channel/v2 v2.0.119/go.mod h1:lSRJwqmbkE34DgXYEmUhVCzwcQcx65vZGE8nuBNK458=
 github.com/openziti/dilithium v0.3.3 h1:PLgQ6PMNLSTzCFbX/h98cmudgz/cU6TmjdSv5NAPD8k=
 github.com/openziti/dilithium v0.3.3/go.mod h1:vsCjI2AU/hon9e+dLhUFbCNGesJDj2ASgkySOcpmvjo=
-github.com/openziti/edge-api v0.26.11 h1:qINsfGpPBTnbuDrOq+qcMZuBdlXosqvHX7sQhLA+cM4=
-github.com/openziti/edge-api v0.26.11/go.mod h1:30SiUmR+9gOBi9HUZgXLpCO2nNCbNFVx2jwXV2Dh4Og=
+github.com/openziti/edge-api v0.26.12 h1:5VRz0cWtfQq2rhSA7Ne6amM7YNI6pQGRfNgbKt0g6kQ=
+github.com/openziti/edge-api v0.26.12/go.mod h1:tKZRUFDB9zM5J1zBS0ok2r40OhJqWykZaU9HSBQgr8w=
 github.com/openziti/foundation/v2 v2.0.37 h1:7pa4vWrlwllEoLXaK2rx91AffLQJ8k5pvc92oWANavA=
 github.com/openziti/foundation/v2 v2.0.37/go.mod h1:2NxzCnJbMw35U9RrFcdEaiXdxIMfBHOUNPngpyhvKeY=
 github.com/openziti/identity v1.0.70 h1:JNwtJHmIS0DcXookm2xuXyh4z92T1O21GQvuO8PmHWs=
@@ -797,8 +797,8 @@ github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQ
 github.com/zitadel/oidc/v2 v2.12.0 h1:4aMTAy99/4pqNwrawEyJqhRb3yY3PtcDxnoDSryhpn4=
 github.com/zitadel/oidc/v2 v2.12.0/go.mod h1:LrRav74IiThHGapQgCHZOUNtnqJG0tcZKHro/91rtLw=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
-go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA=
-go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
+go.etcd.io/bbolt v1.3.9 h1:8x7aARPEXiXbHmtUwAIv7eV2fQFHrLLavdiJ3uzJXoI=
+go.etcd.io/bbolt v1.3.9/go.mod h1:zaO32+Ti0PK1ivdPtgMESzuzL2VPoIG1PCQNvOdo/dE=
 go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
 go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
 go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ=
@@ -814,14 +814,14 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
 go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
-go.opentelemetry.io/otel v1.23.1 h1:Za4UzOqJYS+MUczKI320AtqZHZb7EqxO00jAHE0jmQY=
-go.opentelemetry.io/otel v1.23.1/go.mod h1:Td0134eafDLcTS4y+zQ26GE8u3dEuRBiBCTUIRHaikA=
-go.opentelemetry.io/otel/metric v1.23.1 h1:PQJmqJ9u2QaJLBOELl1cxIdPcpbwzbkjfEyelTl2rlo=
-go.opentelemetry.io/otel/metric v1.23.1/go.mod h1:mpG2QPlAfnK8yNhNJAxDZruU9Y1/HubbC+KyH8FaCWI=
+go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo=
+go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo=
+go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI=
+go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco=
 go.opentelemetry.io/otel/sdk v1.17.0 h1:FLN2X66Ke/k5Sg3V623Q7h7nt3cHXaW1FOvKKrW0IpE=
 go.opentelemetry.io/otel/sdk v1.17.0/go.mod h1:U87sE0f5vQB7hwUoW98pW5Rz4ZDuCFBZFNUBlSgmDFQ=
-go.opentelemetry.io/otel/trace v1.23.1 h1:4LrmmEd8AU2rFvU1zegmvqW7+kWarxtNOPyeL6HmYY8=
-go.opentelemetry.io/otel/trace v1.23.1/go.mod h1:4IpnpJFwr1mo/6HL8XIPJaE9y0+u1KcVmuW7dwFSVrI=
+go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI=
+go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
@@ -849,7 +849,6 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@@ -858,8 +857,9 @@ golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
-golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
 golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
+golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
+golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1072,16 +1072,18 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
 golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
-golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U=
 golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
+golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8=
+golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
diff --git a/router/config.go b/router/config.go
index 29340436a..c9e09358d 100644
--- a/router/config.go
+++ b/router/config.go
@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"fmt"
 	"github.com/openziti/transport/v2/tls"
+	"github.com/openziti/ziti/controller/command"
 	"github.com/openziti/ziti/router/env"
 	"io"
 	"os"
@@ -61,6 +62,23 @@ const (
 
 	// CtrlEndpointBindMapKey is the string key for the ctrl.bind section
 	CtrlEndpointBindMapKey = "bind"
+
+	// CtrlRateLimiterMinSizeValue is the minimum size that can be configured for the control channel rate limiter
+	// window range
+	CtrlRateLimiterMinSizeValue = 5
+
+	// CtrlRateLimiterMaxSizeValue is the maximum size that can be configured for the control channel rate limiter
+	// window range
+	CtrlRateLimiterMaxSizeValue = 1000
+
+	// CtrlRateLimiterMetricOutstandingCount is the name of the metric tracking how many tasks are in process
+	CtrlRateLimiterMetricOutstandingCount = "ctrl.limiter.in_process"
+
+	// CtrlRateLimiterMetricCurrentWindowSize is the name of the metric tracking the current window size
+	CtrlRateLimiterMetricCurrentWindowSize = "ctrl.limiter.window_size"
+
+	// CtrlRateLimiterMetricWorkTimer is the name of the metric tracking how long successful tasks are taking to complete
+	CtrlRateLimiterMetricWorkTimer = "ctrl.limiter.work_timer"
 )
 
 // internalConfigKeys is used to distinguish internally defined configuration vs file configuration
@@ -115,6 +133,7 @@ type Config struct {
 		DataDir               string
 		Heartbeats            env.HeartbeatOptions
 		StartupTimeout        time.Duration
+		RateLimit             command.AdaptiveRateLimiterConfig
 	}
 	Link struct {
 		Listeners  []map[interface{}]interface{}
@@ -527,6 +546,9 @@ func LoadConfig(path string) (*Config, error) {
 			} else {
 				cfg.Ctrl.DataDir = filepath.Dir(cfg.path)
 			}
+			if err = cfg.loadCtrlRateLimiterConfig(submap); err != nil {
+				return nil, err
+			}
 		}
 	}
 
@@ -767,6 +789,43 @@ func LoadConfig(path string) (*Config, error) {
 	return cfg, nil
 }
 
+func (c *Config) loadCtrlRateLimiterConfig(cfgmap map[interface{}]interface{}) error {
+	rateLimitConfig := &c.Ctrl.RateLimit
+	rateLimitConfig.SetDefaults()
+	rateLimitConfig.QueueSizeMetric = CtrlRateLimiterMetricOutstandingCount
+	rateLimitConfig.WindowSizeMetric = CtrlRateLimiterMetricCurrentWindowSize
+	rateLimitConfig.WorkTimerMetric = CtrlRateLimiterMetricWorkTimer
+
+	if value, found := cfgmap["rateLimiter"]; found {
+		if submap, ok := value.(map[interface{}]interface{}); ok {
+			if err := command.LoadAdaptiveRateLimiterConfig(rateLimitConfig, submap); err != nil {
+				return err
+			}
+			if rateLimitConfig.MaxSize < CtrlRateLimiterMinSizeValue {
+				return errors.Errorf("invalid value %v for ctrl.rateLimiter.maxSize, must be at least %v",
+					rateLimitConfig.MaxSize, CtrlRateLimiterMinSizeValue)
+			}
+			if rateLimitConfig.MaxSize > CtrlRateLimiterMaxSizeValue {
+				return errors.Errorf("invalid value %v for ctrl.rateLimiter.maxSize, must be at most %v",
+					rateLimitConfig.MaxSize, CtrlRateLimiterMaxSizeValue)
+			}
+
+			if rateLimitConfig.MinSize < CtrlRateLimiterMinSizeValue {
+				return errors.Errorf("invalid value %v for ctrl.rateLimiter.minSize, must be at least %v",
+					rateLimitConfig.MinSize, CtrlRateLimiterMinSizeValue)
+			}
+			if rateLimitConfig.MinSize > CtrlRateLimiterMaxSizeValue {
+				return errors.Errorf("invalid value %v for ctrl.rateLimiter.minSize, must be at most %v",
+					rateLimitConfig.MinSize, CtrlRateLimiterMaxSizeValue)
+			}
+		} else {
+			return errors.Errorf("invalid type for ctrl.rateLimiter, should be map instead of %T", value)
+		}
+	}
+
+	return nil
+}
+
 func LoadIdentityConfigFromMap(cfgmap map[interface{}]interface{}) (*identity.Config, error) {
 	if value, found := cfgmap["identity"]; found {
 		subMap := value.(map[interface{}]interface{})
diff --git a/router/env/env.go b/router/env/env.go
index 2dc39f18b..0fbe75533 100644
--- a/router/env/env.go
+++ b/router/env/env.go
@@ -22,6 +22,7 @@ import (
 	"github.com/openziti/foundation/v2/versions"
 	"github.com/openziti/identity"
 	"github.com/openziti/metrics"
+	"github.com/openziti/ziti/controller/command"
 	"github.com/openziti/ziti/router/xgress"
 	"github.com/openziti/ziti/router/xlink"
 )
@@ -39,5 +40,6 @@ type RouterEnv interface {
 	RenderJsonConfig() (string, error)
 	GetHeartbeatOptions() HeartbeatOptions
 	GetRateLimiterPool() goroutines.Pool
+	GetCtrlRateLimiter() command.AdaptiveRateLimitTracker
 	GetVersionInfo() versions.VersionProvider
 }
diff --git a/router/fabric/manager.go b/router/fabric/manager.go
index 5a6888e83..09b9febba 100644
--- a/router/fabric/manager.go
+++ b/router/fabric/manager.go
@@ -55,6 +55,7 @@ type StateManager interface {
 	RemoveEdgeSession(token string)
 	AddEdgeSessionRemovedListener(token string, callBack func(token string)) RemoveListener
 	WasSessionRecentlyRemoved(token string) bool
+	MarkSessionRecentlyRemoved(token string)
 
 	//ApiSessions
 	GetApiSession(token string) *ApiSession
@@ -289,6 +290,10 @@ func (sm *StateManagerImpl) WasSessionRecentlyRemoved(token string) bool {
 	return sm.recentlyRemovedSessions.Has(token)
 }
 
+func (sm *StateManagerImpl) MarkSessionRecentlyRemoved(token string) {
+	sm.recentlyRemovedSessions.Set(token, time.Now())
+}
+
 func (sm *StateManagerImpl) AddEdgeSessionRemovedListener(token string, callBack func(token string)) RemoveListener {
 	if sm.recentlyRemovedSessions.Has(token) {
 		go callBack(token) // callback can be long process with network traffic. Don't block event processing
@@ -444,7 +449,7 @@ func (sm *StateManagerImpl) flushRecentlyRemoved() {
 	sm.recentlyRemovedSessions.IterCb(func(key string, t time.Time) {
 		remove := false
 
-		if now.Sub(t) >= time.Minute {
+		if now.Sub(t) >= 5*time.Minute {
 			remove = true
 		}
 
diff --git a/router/handler_ctrl/bind.go b/router/handler_ctrl/bind.go
index 1a2c1de74..8846b4eb2 100644
--- a/router/handler_ctrl/bind.go
+++ b/router/handler_ctrl/bind.go
@@ -32,10 +32,11 @@ import (
 )
 
 type bindHandler struct {
-	env                env.RouterEnv
-	forwarder          *forwarder.Forwarder
-	xgDialerPool       goroutines.Pool
-	ctrlAddressUpdater CtrlAddressUpdater
+	env                      env.RouterEnv
+	forwarder                *forwarder.Forwarder
+	xgDialerPool             goroutines.Pool
+	terminatorValidationPool goroutines.Pool
+	ctrlAddressUpdater       CtrlAddressUpdater
 }
 
 func NewBindHandler(routerEnv env.RouterEnv, forwarder *forwarder.Forwarder, ctrlAddressUpdater CtrlAddressUpdater) (channel.BindHandler, error) {
@@ -57,11 +58,30 @@ func NewBindHandler(routerEnv env.RouterEnv, forwarder *forwarder.Forwarder, ctr
 		return nil, errors.Wrap(err, "error creating xgress route handler pool")
 	}
 
+	terminatorValidatorPoolConfig := goroutines.PoolConfig{
+		QueueSize:   uint32(1),
+		MinWorkers:  0,
+		MaxWorkers:  uint32(50),
+		IdleTime:    30 * time.Second,
+		CloseNotify: routerEnv.GetCloseNotify(),
+		PanicHandler: func(err interface{}) {
+			pfxlog.Logger().WithField(logrus.ErrorKey, err).WithField("backtrace", string(debug.Stack())).Error("panic during terminator validation operation")
+		},
+	}
+
+	metrics.ConfigureGoroutinesPoolMetrics(&terminatorValidatorPoolConfig, routerEnv.GetMetricsRegistry(), "pool.terminator_validation")
+
+	terminatorValidationPool, err := goroutines.NewPool(terminatorValidatorPoolConfig)
+	if err != nil {
+		return nil, errors.Wrap(err, "error creating terminator validation pool")
+	}
+
 	return &bindHandler{
-		env:                routerEnv,
-		forwarder:          forwarder,
-		xgDialerPool:       xgDialerPool,
-		ctrlAddressUpdater: ctrlAddressUpdater,
+		env:                      routerEnv,
+		forwarder:                forwarder,
+		xgDialerPool:             xgDialerPool,
+		terminatorValidationPool: terminatorValidationPool,
+		ctrlAddressUpdater:       ctrlAddressUpdater,
 	}, nil
 }
 
@@ -70,7 +90,7 @@ func (self *bindHandler) BindChannel(binding channel.Binding) error {
 	binding.AddTypedReceiveHandler(newDialHandler(self.env))
 	binding.AddTypedReceiveHandler(newRouteHandler(binding.GetChannel(), self.env, self.forwarder, self.xgDialerPool))
 	binding.AddTypedReceiveHandler(newValidateTerminatorsHandler(self.env))
-	binding.AddTypedReceiveHandler(newValidateTerminatorsV2Handler(self.env))
+	binding.AddTypedReceiveHandler(newValidateTerminatorsV2Handler(self.env, self.terminatorValidationPool))
 	binding.AddTypedReceiveHandler(newUnrouteHandler(self.forwarder))
 	binding.AddTypedReceiveHandler(newTraceHandler(self.env.GetRouterId(), self.forwarder.TraceController(), binding.GetChannel()))
 	binding.AddTypedReceiveHandler(newInspectHandler(self.env, self.forwarder))
diff --git a/router/handler_ctrl/inspect.go b/router/handler_ctrl/inspect.go
index b2e1705ba..7434c727e 100644
--- a/router/handler_ctrl/inspect.go
+++ b/router/handler_ctrl/inspect.go
@@ -18,12 +18,14 @@ package handler_ctrl
 
 import (
 	"encoding/json"
+	"fmt"
 	"github.com/michaelquigley/pfxlog"
 	"github.com/openziti/channel/v2"
+	"github.com/openziti/foundation/v2/debugz"
 	"github.com/openziti/ziti/common/pb/ctrl_pb"
 	"github.com/openziti/ziti/router/env"
 	"github.com/openziti/ziti/router/forwarder"
-	"github.com/openziti/foundation/v2/debugz"
+	"github.com/openziti/ziti/router/xgress"
 	"github.com/pkg/errors"
 	"google.golang.org/protobuf/proto"
 	"strings"
@@ -90,6 +92,29 @@ func (context *inspectRequestContext) processLocal() {
 			} else {
 				context.appendValue(requested, string(js))
 			}
+		} else if lc == "sdk-terminators" {
+			factory, _ := xgress.GlobalRegistry().Factory("edge")
+			if factory == nil {
+				context.appendError("no xgress factory configured for edge binding")
+				continue
+			}
+			dialer, err := factory.CreateDialer(context.handler.env.GetDialerCfg()["edge"])
+			if err != nil {
+				context.appendError(fmt.Sprintf("could not create edge dialer: (%s)", err.Error()))
+				continue
+			}
+			inspectable, ok := dialer.(xgress.Inspectable)
+			if !ok {
+				context.appendError("edge dialer is not of type Inspectable")
+				continue
+			}
+			result := inspectable.Inspect(lc, time.Second)
+			js, err := json.Marshal(result)
+			if err != nil {
+				context.appendError(errors.Wrap(err, "failed to marshal sdk terminators to json").Error())
+			} else {
+				context.appendValue(requested, string(js))
+			}
 		} else if strings.HasPrefix(lc, "circuit:") {
 			circuitId := requested[len("circuit:"):]
 			result := context.handler.fwd.InspectCircuit(circuitId, false)
diff --git a/router/handler_ctrl/validate_terminators_v2.go b/router/handler_ctrl/validate_terminators_v2.go
index 0a06666d2..4a0cd9021 100644
--- a/router/handler_ctrl/validate_terminators_v2.go
+++ b/router/handler_ctrl/validate_terminators_v2.go
@@ -20,19 +20,23 @@ import (
 	"github.com/michaelquigley/pfxlog"
 	"github.com/openziti/channel/v2"
 	"github.com/openziti/channel/v2/protobufs"
+	"github.com/openziti/foundation/v2/goroutines"
 	"github.com/openziti/ziti/common/pb/ctrl_pb"
 	"github.com/openziti/ziti/router/env"
 	"github.com/openziti/ziti/router/xgress"
 	"google.golang.org/protobuf/proto"
+	"time"
 )
 
 type validateTerminatorsV2Handler struct {
-	env env.RouterEnv
+	env  env.RouterEnv
+	pool goroutines.Pool
 }
 
-func newValidateTerminatorsV2Handler(env env.RouterEnv) *validateTerminatorsV2Handler {
+func newValidateTerminatorsV2Handler(env env.RouterEnv, pool goroutines.Pool) *validateTerminatorsV2Handler {
 	return &validateTerminatorsV2Handler{
-		env: env,
+		env:  env,
+		pool: pool,
 	}
 }
 
@@ -64,7 +68,11 @@ func (handler *validateTerminatorsV2Handler) validateTerminators(msg *channel.Me
 		States: map[string]*ctrl_pb.RouterTerminatorState{},
 	}
 
-	for _, terminator := range req.Terminators {
+	expected := 0
+	results := make(chan func(*ctrl_pb.ValidateTerminatorsV2Response), len(req.Terminators))
+
+	for _, val := range req.Terminators {
+		terminator := val
 		binding := terminator.Binding
 		dialer := dialers[binding]
 		if dialer == nil {
@@ -74,32 +82,42 @@ func (handler *validateTerminatorsV2Handler) validateTerminators(msg *channel.Me
 				}
 			}
 		}
-
+		log.WithField("terminatorId", terminator.Id).Debug("beginning terminator validation")
 		if dialer == nil {
 			response.States[terminator.Id] = &ctrl_pb.RouterTerminatorState{
 				Valid:  false,
 				Reason: ctrl_pb.TerminatorInvalidReason_UnknownBinding,
-			}
-		} else if inspectable, ok := dialer.(xgress.InspectableDialer); ok {
-			valid, state := inspectable.InspectTerminator(terminator.Id, terminator.Address, req.FixInvalid)
-			response.States[terminator.Id] = &ctrl_pb.RouterTerminatorState{
-				Valid:  valid,
-				Detail: state,
-				Reason: ctrl_pb.TerminatorInvalidReason_UnknownTerminator,
-			}
-		} else if !dialer.IsTerminatorValid(terminator.Id, terminator.Address) {
-			response.States[terminator.Id] = &ctrl_pb.RouterTerminatorState{
-				Valid:  false,
-				Reason: ctrl_pb.TerminatorInvalidReason_UnknownTerminator,
+				Marker: val.Marker,
 			}
 		} else {
-			response.States[terminator.Id] = &ctrl_pb.RouterTerminatorState{
-				Valid:  true,
-				Detail: "valid",
+			err := handler.pool.Queue(func() {
+				log.WithField("terminatorId", terminator.Id).Info("validating terminator")
+				result := handler.validateTerminator(dialer, terminator, req.FixInvalid)
+				results <- func(response *ctrl_pb.ValidateTerminatorsV2Response) {
+					response.States[terminator.Id] = result
+				}
+			})
+
+			if err != nil {
+				log.WithField("terminatorId", terminator.Id).WithError(err).Error("unable to queue inspect")
+			} else {
+				expected++
 			}
 		}
 	}
 
+	timeout := time.After(30 * time.Second)
+	timedOut := false
+	for i := 0; i < expected && !timedOut; i++ {
+		select {
+		case result := <-results:
+			result(response)
+		case <-timeout:
+			timedOut = true
+			log.Info("timed out waiting for terminator validations")
+		}
+	}
+
 	err := protobufs.MarshalTyped(response).
 		ReplyTo(msg).
 		WithTimeout(handler.env.GetNetworkControllers().DefaultRequestTimeout()).
@@ -109,3 +127,29 @@ func (handler *validateTerminatorsV2Handler) validateTerminators(msg *channel.Me
 		log.WithError(err).Error("failed to send validate terminators v2 response")
 	}
 }
+
+func (handler *validateTerminatorsV2Handler) validateTerminator(dialer xgress.Dialer, terminator *ctrl_pb.Terminator, fixInvalid bool) *ctrl_pb.RouterTerminatorState {
+	if inspectable, ok := dialer.(xgress.InspectableDialer); ok {
+		valid, state := inspectable.InspectTerminator(terminator.Id, terminator.Address, fixInvalid)
+		return &ctrl_pb.RouterTerminatorState{
+			Valid:  valid,
+			Detail: state,
+			Reason: ctrl_pb.TerminatorInvalidReason_UnknownTerminator,
+			Marker: terminator.Marker,
+		}
+	}
+
+	if !dialer.IsTerminatorValid(terminator.Id, terminator.Address) {
+		return &ctrl_pb.RouterTerminatorState{
+			Valid:  false,
+			Reason: ctrl_pb.TerminatorInvalidReason_UnknownTerminator,
+			Marker: terminator.Marker,
+		}
+	}
+
+	return &ctrl_pb.RouterTerminatorState{
+		Valid:  true,
+		Detail: "valid",
+		Marker: terminator.Marker,
+	}
+}
diff --git a/router/router.go b/router/router.go
index 593dc1e39..6c9ebcaa4 100644
--- a/router/router.go
+++ b/router/router.go
@@ -21,6 +21,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"github.com/openziti/ziti/controller/command"
 	"io/fs"
 	"os"
 	"path"
@@ -82,6 +83,7 @@ type Router struct {
 	xgressListeners []xgress.Listener
 	linkDialerPool  goroutines.Pool
 	rateLimiterPool goroutines.Pool
+	ctrlRateLimiter command.AdaptiveRateLimitTracker
 	metricsRegistry metrics.UsageRegistry
 	shutdownC       chan struct{}
 	shutdownDoneC   chan struct{}
@@ -191,6 +193,7 @@ func Create(config *Config, versionProvider versions.VersionProvider) *Router {
 		debugOperations:     map[byte]func(c *bufio.ReadWriter) error{},
 		xwebFactoryRegistry: xweb.NewRegistryMap(),
 		linkDialerPool:      linkDialerPool,
+		ctrlRateLimiter:     command.NewAdaptiveRateLimitTracker(config.Ctrl.RateLimit, metricsRegistry, closeNotify),
 	}
 
 	router.ctrls = env.NewNetworkControllers(config.Ctrl.DefaultRequestTimeout, router.connectToController, &config.Ctrl.Heartbeats)
@@ -379,6 +382,10 @@ func (self *Router) GetRateLimiterPool() goroutines.Pool {
 	return self.rateLimiterPool
 }
 
+func (self *Router) GetCtrlRateLimiter() command.AdaptiveRateLimitTracker {
+	return self.ctrlRateLimiter
+}
+
 func (self *Router) registerComponents() error {
 	self.xlinkFactories = make(map[string]xlink.Factory)
 	xlinkAccepter := newXlinkAccepter(self.forwarder)
diff --git a/router/router_test.go b/router/router_test.go
index d2fce62f7..53fdd30ab 100644
--- a/router/router_test.go
+++ b/router/router_test.go
@@ -1,6 +1,7 @@
 package router
 
 import (
+	"github.com/openziti/ziti/controller/command"
 	"github.com/openziti/ziti/router/env"
 	"github.com/stretchr/testify/require"
 	"os"
@@ -50,6 +51,7 @@ func Test_initializeCtrlEndpoints(t *testing.T) {
 				DataDir               string
 				Heartbeats            env.HeartbeatOptions
 				StartupTimeout        time.Duration
+				RateLimit             command.AdaptiveRateLimiterConfig
 			}{
 				DataDir:          tmpDir,
 				InitialEndpoints: []*UpdatableAddress{NewUpdatableAddress(addr)},
@@ -91,6 +93,7 @@ func Test_updateCtrlEndpoints(t *testing.T) {
 				DataDir               string
 				Heartbeats            env.HeartbeatOptions
 				StartupTimeout        time.Duration
+				RateLimit             command.AdaptiveRateLimiterConfig
 			}{
 				DataDir:          tmpDir,
 				InitialEndpoints: []*UpdatableAddress{NewUpdatableAddress(addr), NewUpdatableAddress(addr2)},
diff --git a/router/xgress/request.go b/router/xgress/request.go
index d0e1e59ab..6dbbe7c48 100644
--- a/router/xgress/request.go
+++ b/router/xgress/request.go
@@ -21,10 +21,10 @@ import (
 	"github.com/michaelquigley/pfxlog"
 	"github.com/openziti/channel/v2"
 	"github.com/openziti/channel/v2/protobufs"
-	"github.com/openziti/ziti/common/ctrl_msg"
-	"github.com/openziti/ziti/common/pb/ctrl_pb"
 	"github.com/openziti/identity"
 	"github.com/openziti/transport/v2"
+	"github.com/openziti/ziti/common/ctrl_msg"
+	"github.com/openziti/ziti/common/pb/ctrl_pb"
 	"github.com/pkg/errors"
 	"io"
 	"time"
diff --git a/router/xgress/xgress.go b/router/xgress/xgress.go
index b6ae4b170..d876e4855 100644
--- a/router/xgress/xgress.go
+++ b/router/xgress/xgress.go
@@ -77,6 +77,10 @@ type InspectableDialer interface {
 	InspectTerminator(id string, destination string, fixInvalid bool) (bool, string)
 }
 
+type Inspectable interface {
+	Inspect(key string, timeout time.Duration) any
+}
+
 type Factory interface {
 	CreateListener(optionsData OptionsData) (Listener, error)
 	CreateDialer(optionsData OptionsData) (Dialer, error)
diff --git a/router/xgress_edge/dialer.go b/router/xgress_edge/dialer.go
index 13a31b6c2..c9839fec9 100644
--- a/router/xgress_edge/dialer.go
+++ b/router/xgress_edge/dialer.go
@@ -44,8 +44,9 @@ func (dialer *dialer) InspectTerminator(id string, destination string, fixInvali
 	terminatorAddress := strings.TrimPrefix(destination, "hosted:")
 	pfxlog.Logger().Debug("looking up hosted service conn")
 	terminator, found := dialer.factory.hostedServices.Get(terminatorAddress)
-	if found && terminator.terminatorId.Load() == id {
-		result, err := terminator.inspect(fixInvalid)
+	if found && terminator.terminatorId == id {
+		dialer.factory.hostedServices.markEstablished(terminator, "validation message received")
+		result, err := terminator.inspect(dialer.factory.hostedServices, fixInvalid, false)
 		if err != nil {
 			return true, err.Error()
 		}
@@ -187,3 +188,10 @@ func (dialer *dialer) Dial(params xgress.DialParams) (xt.PeerData, error) {
 		return nil, terminator.SendState(start)
 	}
 }
+
+func (dialer *dialer) Inspect(key string, timeout time.Duration) any {
+	if key == "sdk-terminators" {
+		return dialer.factory.hostedServices.Inspect(timeout)
+	}
+	return nil
+}
diff --git a/router/xgress_edge/fabric.go b/router/xgress_edge/fabric.go
index e1e35a981..861e6d657 100644
--- a/router/xgress_edge/fabric.go
+++ b/router/xgress_edge/fabric.go
@@ -17,17 +17,19 @@
 package xgress_edge
 
 import (
+	"fmt"
 	"github.com/michaelquigley/pfxlog"
 	"github.com/openziti/channel/v2"
 	"github.com/openziti/foundation/v2/concurrenz"
 	"github.com/openziti/sdk-golang/ziti/edge"
 	"github.com/openziti/ziti/common/pb/edge_ctrl_pb"
+	"github.com/openziti/ziti/controller/command"
 	"github.com/openziti/ziti/router/xgress"
 	"github.com/openziti/ziti/router/xgress_common"
 	"github.com/pkg/errors"
 	"io"
 	"math"
-	"math/rand"
+	"sync"
 	"sync/atomic"
 	"time"
 )
@@ -44,16 +46,13 @@ var headersFromFabric = map[uint8]int32{
 type terminatorState int
 
 const (
-	TerminatorStatePendingEstablishment terminatorState = 0
-	TerminatorStateEstablishing         terminatorState = 1
-	TerminatorStateEstablished          terminatorState = 2
-	TerminatorStateDeleting             terminatorState = 3
+	TerminatorStateEstablishing terminatorState = 1
+	TerminatorStateEstablished  terminatorState = 2
+	TerminatorStateDeleting     terminatorState = 3
 )
 
 func (self terminatorState) String() string {
 	switch self {
-	case TerminatorStatePendingEstablishment:
-		return "pending-establishment"
 	case TerminatorStateEstablishing:
 		return "establishing"
 	case TerminatorStateEstablished:
@@ -65,10 +64,23 @@ func (self terminatorState) String() string {
 	}
 }
 
+func (self terminatorState) IsWorkRequired() bool {
+	switch self {
+	case TerminatorStateEstablishing:
+		return true
+	case TerminatorStateDeleting:
+		return true
+	case TerminatorStateEstablished:
+		return false
+	default:
+		return false
+	}
+}
+
 type edgeTerminator struct {
 	edge.MsgChannel
 	edgeClientConn    *edgeClientConn
-	terminatorId      concurrenz.AtomicValue[string]
+	terminatorId      string
 	listenerId        string
 	token             string
 	instance          string
@@ -80,44 +92,113 @@ type edgeTerminator struct {
 	onClose           func()
 	v2                bool
 	state             concurrenz.AtomicValue[terminatorState]
-	postValidate      bool
-	nextAttempt       time.Time
-	retryDelay        time.Duration
-	establishActive   atomic.Bool
+	supportsInspect   bool
+	operationActive   atomic.Bool
 	createTime        time.Time
-	establishCallback func(ok bool, msg string)
+	lastAttempt       time.Time
+	establishCallback func(result edge_ctrl_pb.CreateTerminatorResult)
+	lock              sync.Mutex
+	rateLimitCallback command.RateLimitControl
 }
 
-func (self *edgeTerminator) calculateRetry(queueFailed bool) {
-	retryDelay := time.Second
+func (self *edgeTerminator) replace(other *edgeTerminator) {
+	other.lock.Lock()
+	terminatorId := other.terminatorId
+	state := other.state.Load()
+	rateLimitCallback := other.rateLimitCallback
+	operationActive := other.operationActive.Load()
+	createTime := other.createTime
+	lastAttempt := other.lastAttempt
+	other.lock.Unlock()
+
+	self.lock.Lock()
+	self.terminatorId = terminatorId
+	self.setState(state, "replacing existing terminator")
+	self.rateLimitCallback = rateLimitCallback
+	self.operationActive.Store(operationActive)
+	self.createTime = createTime
+	self.lastAttempt = lastAttempt
+	self.lock.Unlock()
+}
 
-	if !queueFailed {
-		if self.retryDelay < time.Second {
-			self.retryDelay = time.Second
-		} else {
-			self.retryDelay = self.retryDelay * 2
-			if self.retryDelay > 30*time.Second {
-				self.retryDelay = 30 * time.Second
-			}
-		}
-		retryDelay = self.retryDelay
+func (self *edgeTerminator) IsEstablishing() bool {
+	return self.state.Load() == TerminatorStateEstablishing
+}
+
+func (self *edgeTerminator) IsDeleting() bool {
+	return self.state.Load() == TerminatorStateDeleting
+}
+
+func (self *edgeTerminator) setState(state terminatorState, reason string) {
+	if oldState := self.state.Load(); oldState != state {
+		self.state.Store(state)
+		pfxlog.Logger().WithField("terminatorId", self.terminatorId).
+			WithField("oldState", oldState).
+			WithField("newState", state).
+			WithField("reason", reason).
+			Info("updated state")
 	}
+}
 
-	actualDelay := (float64(retryDelay.Milliseconds()) * 1.5) - float64(rand.Intn(int(retryDelay.Milliseconds())))
-	self.nextAttempt = time.Now().Add(time.Millisecond * time.Duration(actualDelay))
+func (self *edgeTerminator) updateState(oldState, newState terminatorState, reason string) bool {
+	log := pfxlog.Logger().WithField("terminatorId", self.terminatorId).
+		WithField("oldState", oldState).
+		WithField("newState", newState).
+		WithField("reason", reason)
+	success := self.state.CompareAndSwap(oldState, newState)
+	if success {
+		log.Info("updated state")
+	}
+	return success
 }
 
-func (self *edgeTerminator) inspect(fixInvalidTerminators bool) (*edge.InspectResult, error) {
-	msg := channel.NewMessage(edge.ContentTypeConnInspectRequest, nil)
-	msg.PutUint32Header(edge.ConnIdHeader, self.Id())
-	resp, err := msg.WithTimeout(10 * time.Second).SendForReply(self.Channel)
-	if err != nil {
-		return nil, errors.New("unable to check status with sdk client")
+func (self *edgeTerminator) inspect(registry *hostedServiceRegistry, fixInvalidTerminators bool, notifyCtrl bool) (*edge.InspectResult, error) {
+	result := &edge.InspectResult{
+		ConnId: self.MsgChannel.Id(),
+		Type:   edge.ConnTypeUnknown,
+		Detail: "channel closed",
 	}
-	result, err := edge.UnmarshalInspectResult(resp)
-	if result != nil && result.Type != edge.ConnTypeBind && fixInvalidTerminators {
-		self.close(true, true, "terminator invalid")
+
+	var err error
+	isInvalid := false
+	if !self.Channel.IsClosed() {
+		msg := channel.NewMessage(edge.ContentTypeConnInspectRequest, nil)
+		msg.PutUint32Header(edge.ConnIdHeader, self.Id())
+		resp, err := msg.WithTimeout(10 * time.Second).SendForReply(self.Channel)
+		if err != nil {
+			return nil, fmt.Errorf("unable to check status with sdk client: (%w)", err)
+		}
+
+		result, err = edge.UnmarshalInspectResult(resp)
+		if err != nil {
+			return nil, fmt.Errorf("unable to unmarshal inspect response from sdk client: (%w)", err)
+		}
+
+		isInvalid = result != nil && result.Type != edge.ConnTypeBind
+	} else {
+		isInvalid = true
+		err = errors.New("channel closed")
 	}
+
+	if isInvalid && fixInvalidTerminators {
+		removeResult := registry.handleSdkReturnedInvalid(self, notifyCtrl)
+		if removeResult.err != nil {
+			return nil, err
+		}
+		if removeResult.removed || !removeResult.existed {
+			return result, err
+		}
+		current, _ := registry.Get(self.terminatorId)
+		if current == nil {
+			return result, err
+		}
+
+		if current == self { // this shouldn't happen
+			return result, errors.New("wasn't able to remove, but is still current")
+		}
+		return current.inspect(registry, fixInvalidTerminators, notifyCtrl)
+	}
+
 	return result, err
 }
 
@@ -130,9 +211,9 @@ func (self *edgeTerminator) nextDialConnId() uint32 {
 	return nextId
 }
 
-func (self *edgeTerminator) close(notifySdk bool, notifyCtrl bool, reason string) {
+func (self *edgeTerminator) close(registry *hostedServiceRegistry, notifySdk bool, notifyCtrl bool, reason string) {
 	logger := pfxlog.Logger().
-		WithField("terminatorId", self.terminatorId.Load()).
+		WithField("terminatorId", self.terminatorId).
 		WithField("token", self.token).
 		WithField("reason", reason)
 
@@ -146,36 +227,19 @@ func (self *edgeTerminator) close(notifySdk bool, notifyCtrl bool, reason string
 	}
 
 	if self.v2 {
-		if terminatorId := self.terminatorId.Load(); terminatorId != "" {
-			if self.terminatorId.CompareAndSwap(terminatorId, "") {
-				logger.Debug("removing terminator on router")
-
-				self.state.Store(TerminatorStateDeleting)
-				self.edgeClientConn.listener.factory.hostedServices.Delete(terminatorId)
-
-				if notifyCtrl {
-					logger.Info("removing terminator on controller")
-					ctrlCh := self.edgeClientConn.listener.factory.ctrls.AnyCtrlChannel()
-					if ctrlCh == nil {
-						logger.Error("no controller available, unable to remove terminator")
-					} else if err := self.edgeClientConn.removeTerminator(ctrlCh, self.token, terminatorId); err != nil {
-						logger.WithError(err).Error("failed to remove terminator")
-					} else {
-						logger.Info("Successfully removed terminator")
-					}
-				}
-			} else {
-				logger.Warn("edge terminator closing, but no terminator id set, so can't remove on controller")
-			}
+		if notifyCtrl {
+			registry.queueRemoveTerminatorAsync(self, reason)
+		} else {
+			self.edgeClientConn.listener.factory.hostedServices.Remove(self, reason)
 		}
 	} else {
 		if notifyCtrl {
-			if terminatorId := self.terminatorId.Load(); terminatorId != "" {
+			if self.terminatorId != "" {
 				logger.Info("removing terminator on controller")
 				ctrlCh := self.edgeClientConn.listener.factory.ctrls.AnyCtrlChannel()
 				if ctrlCh == nil {
 					logger.Error("no controller available, unable to remove terminator")
-				} else if err := self.edgeClientConn.removeTerminator(ctrlCh, self.token, terminatorId); err != nil {
+				} else if err := self.edgeClientConn.removeTerminator(ctrlCh, self.token, self.terminatorId); err != nil {
 					logger.WithError(err).Error("failed to remove terminator")
 				} else {
 					logger.Info("successfully removed terminator")
@@ -185,7 +249,7 @@ func (self *edgeTerminator) close(notifySdk bool, notifyCtrl bool, reason string
 			}
 		}
 
-		logger.Debug("removing terminator on router")
+		logger.Info("terminator removed from router set")
 		self.edgeClientConn.listener.factory.hostedServices.Delete(self.token)
 	}
 
@@ -209,6 +273,20 @@ func (self *edgeTerminator) newConnection(connId uint32) (*edgeXgressConn, error
 	return result, nil
 }
 
+func (self *edgeTerminator) SetRateLimitCallback(control command.RateLimitControl) {
+	self.lock.Lock()
+	defer self.lock.Unlock()
+	self.rateLimitCallback = control
+}
+
+func (self *edgeTerminator) GetAndClearRateLimitCallback() command.RateLimitControl {
+	self.lock.Lock()
+	defer self.lock.Unlock()
+	result := self.rateLimitCallback
+	self.rateLimitCallback = nil
+	return result
+}
+
 type edgeXgressConn struct {
 	edge.MsgChannel
 	mux     edge.MsgMux
diff --git a/router/xgress_edge/hosted.go b/router/xgress_edge/hosted.go
index ab19dbe2f..ddf64f75c 100644
--- a/router/xgress_edge/hosted.go
+++ b/router/xgress_edge/hosted.go
@@ -17,39 +17,45 @@
 package xgress_edge
 
 import (
-	"container/heap"
 	"fmt"
 	"github.com/michaelquigley/pfxlog"
 	"github.com/openziti/channel/v2"
 	"github.com/openziti/channel/v2/protobufs"
-	"github.com/openziti/sdk-golang/ziti/edge"
+	"github.com/openziti/ziti/common/handler_common"
+	"github.com/openziti/ziti/common/inspect"
+	"github.com/openziti/ziti/common/pb/ctrl_pb"
 	"github.com/openziti/ziti/common/pb/edge_ctrl_pb"
+	"github.com/openziti/ziti/controller/apierror"
+	"github.com/openziti/ziti/controller/command"
 	routerEnv "github.com/openziti/ziti/router/env"
+	cmap "github.com/orcaman/concurrent-map/v2"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"google.golang.org/protobuf/proto"
-	"sync"
+	"sync/atomic"
 	"time"
 )
 
 func newHostedServicesRegistry(env routerEnv.RouterEnv) *hostedServiceRegistry {
 	result := &hostedServiceRegistry{
-		services:        sync.Map{},
-		events:          make(chan terminatorEvent),
-		env:             env,
-		retriesPending:  false,
-		terminatorQueue: &terminatorHeap{},
+		terminators:  cmap.New[*edgeTerminator](),
+		events:       make(chan terminatorEvent),
+		env:          env,
+		triggerEvalC: make(chan struct{}, 1),
+		establishSet: map[string]*edgeTerminator{},
+		deleteSet:    map[string]*edgeTerminator{},
 	}
 	go result.run()
 	return result
 }
 
 type hostedServiceRegistry struct {
-	services        sync.Map
-	events          chan terminatorEvent
-	env             routerEnv.RouterEnv
-	retriesPending  bool
-	terminatorQueue *terminatorHeap
+	terminators  cmap.ConcurrentMap[string, *edgeTerminator]
+	events       chan terminatorEvent
+	env          routerEnv.RouterEnv
+	establishSet map[string]*edgeTerminator
+	deleteSet    map[string]*edgeTerminator
+	triggerEvalC chan struct{}
 }
 
 type terminatorEvent interface {
@@ -57,67 +63,290 @@ type terminatorEvent interface {
 }
 
 func (self *hostedServiceRegistry) run() {
-	queueCheckTicker := time.NewTicker(100 * time.Millisecond)
-	defer queueCheckTicker.Stop()
-
-	longQueueCheckTicker := time.NewTicker(time.Second)
+	longQueueCheckTicker := time.NewTicker(time.Minute)
 	defer longQueueCheckTicker.Stop()
 
+	quickTick := time.NewTicker(10 * time.Millisecond)
+	defer quickTick.Stop()
+
 	for {
-		var retryChan <-chan time.Time
-		if self.retriesPending {
-			retryChan = queueCheckTicker.C
+		var rateLimitedTick <-chan time.Time
+		if self.env.GetCtrlRateLimiter().IsRateLimited() {
+			rateLimitedTick = quickTick.C
 		}
-
 		select {
 		case <-self.env.GetCloseNotify():
 			return
 		case event := <-self.events:
 			event.handle(self)
-		case <-retryChan:
-			self.evaluateLinkStateQueue()
 		case <-longQueueCheckTicker.C:
 			self.scanForRetries()
+		case <-self.triggerEvalC:
+		case <-rateLimitedTick:
+		}
+
+		// events should be quick to handle, so make sure we do all them before we
+		// try the establish/delete queues
+		allEventsHandled := false
+		for !allEventsHandled {
+			select {
+			case event := <-self.events:
+				event.handle(self)
+			default:
+				allEventsHandled = true
+			}
+		}
+
+		if !self.env.GetCtrlRateLimiter().IsRateLimited() {
+			self.evaluateEstablishQueue()
+		}
+
+		if !self.env.GetCtrlRateLimiter().IsRateLimited() {
+			self.evaluateDeleteQueue()
 		}
 	}
 }
 
-func (self *hostedServiceRegistry) evaluateLinkStateQueue() {
-	now := time.Now()
-	for len(*self.terminatorQueue) > 0 {
-		next := (*self.terminatorQueue)[0]
-		if now.Before(next.nextAttempt) {
+func (self *hostedServiceRegistry) triggerEvaluates() {
+	select {
+	case self.triggerEvalC <- struct{}{}:
+	default:
+	}
+}
+
+func (self *hostedServiceRegistry) evaluateEstablishQueue() {
+	for id, terminator := range self.establishSet {
+		dequeue := func() {
+			delete(self.establishSet, id)
+		}
+
+		log := logrus.
+			WithField("terminatorId", terminator.terminatorId).
+			WithField("state", terminator.state.Load()).
+			WithField("token", terminator.token)
+
+		if terminator.edgeClientConn.ch.IsClosed() {
+			self.Remove(terminator, "sdk connection is closed")
+			log.Infof("terminator sdk channel closed, not trying to establish")
+			dequeue()
+			continue
+		}
+
+		if terminator.terminatorId == "" {
+			log.Info("terminator has been closed, not trying to establish")
+			dequeue()
+			continue
+		}
+
+		if terminator.state.Load() != TerminatorStateEstablishing {
+			dequeue()
+			continue
+		}
+
+		label := fmt.Sprintf("establish terminator %s", terminator.terminatorId)
+		rateLimitCtrl, err := self.env.GetCtrlRateLimiter().RunRateLimited(label)
+		if err != nil {
+			log.Info("rate limiter hit, waiting for a slot to open")
 			return
 		}
-		heap.Pop(self.terminatorQueue)
-		self.evaluateTerminator(next)
+
+		if !terminator.operationActive.CompareAndSwap(false, true) && time.Since(terminator.lastAttempt) < 30*time.Second {
+			rateLimitCtrl.Failed()
+			continue
+		}
+
+		log.Info("queuing terminator to send create")
+
+		dequeue()
+		terminator.SetRateLimitCallback(rateLimitCtrl)
+		terminator.lastAttempt = time.Now()
+
+		if err = self.establishTerminator(terminator); err != nil {
+			log.WithError(err).Error("error establishing terminator")
+			self.queueEstablishTerminatorSync(terminator)
+			rateLimitCtrl.Failed()
+			return // if we had an error it's because the channel was busy or a controller wasn't available
+		}
 	}
-	self.retriesPending = false
 }
 
-type establishTerminatorEvent struct {
+func (self *hostedServiceRegistry) evaluateDeleteQueue() {
+	var deleteList []*edgeTerminator
+
+	for terminatorId, terminator := range self.deleteSet {
+		log := logrus.
+			WithField("terminatorId", terminator.terminatorId).
+			WithField("state", terminator.state.Load()).
+			WithField("token", terminator.token)
+
+		delete(self.deleteSet, terminatorId)
+
+		if terminator.state.Load() != TerminatorStateDeleting {
+			continue
+		}
+
+		if current, exists := self.terminators.Get(terminatorId); exists && current != terminator {
+			continue
+		}
+
+		if terminator.operationActive.Load() {
+			if time.Since(terminator.lastAttempt) > 30*time.Second {
+				terminator.operationActive.Store(false)
+			} else {
+				continue
+			}
+		}
+
+		log.Info("added terminator to batch delete")
+		deleteList = append(deleteList, terminator)
+		if len(deleteList) >= 50 {
+			if !self.RemoveTerminatorsRateLimited(deleteList) {
+				return
+			}
+			deleteList = nil
+		}
+	}
+
+	if len(deleteList) != 0 {
+		self.RemoveTerminatorsRateLimited(deleteList)
+	}
+}
+
+func (self *hostedServiceRegistry) RemoveTerminatorsRateLimited(terminators []*edgeTerminator) bool {
+	if self.env.GetCtrlRateLimiter().IsRateLimited() {
+		self.requeueForDeleteSync(terminators)
+		return false
+	}
+
+	for _, terminator := range terminators {
+		terminator.operationActive.Store(true)
+	}
+
+	err := self.env.GetRateLimiterPool().QueueOrError(func() {
+		rateLimitCtrl, err := self.env.GetCtrlRateLimiter().RunRateLimited("remove terminator batch")
+		if err != nil {
+			pfxlog.Logger().Debug("rate limiter hit, waiting for a slot to open before doing sdk terminator deletes")
+
+			for _, terminator := range terminators {
+				pfxlog.Logger().WithError(err).WithField("terminatorId", terminator.terminatorId).
+					Error("remove terminator failed")
+				self.requeueRemoveTerminatorAsync(terminator)
+			}
+			return
+		}
+
+		var terminatorIds []string
+		for _, terminator := range terminators {
+			terminatorIds = append(terminatorIds, terminator.terminatorId)
+		}
+
+		if err := self.RemoveTerminators(terminatorIds); err != nil {
+			if command.WasRateLimited(err) {
+				rateLimitCtrl.Backoff()
+			} else {
+				rateLimitCtrl.Failed()
+			}
+
+			for _, terminator := range terminators {
+				pfxlog.Logger().WithError(err).WithField("terminatorId", terminator.terminatorId).
+					Error("remove terminator failed")
+				self.requeueRemoveTerminatorAsync(terminator)
+			}
+		} else {
+			rateLimitCtrl.Success()
+			for _, terminator := range terminators {
+				pfxlog.Logger().WithField("terminatorId", terminator.terminatorId).
+					Info("remove terminator succeeded")
+				terminator.operationActive.Store(false)
+				if !self.Remove(terminator, "controller delete success") {
+					pfxlog.Logger().WithField("terminatorId", terminator.terminatorId).
+						Error("terminator was replaced after being put into deleting state?!")
+				}
+			}
+		}
+	})
+
+	if err != nil {
+		pfxlog.Logger().WithError(err).Error("unable to queue remove terminators operation")
+		self.requeueForDeleteSync(terminators)
+		return false
+	}
+
+	return true
+}
+
+func (self *hostedServiceRegistry) requeueForDeleteSync(terminators []*edgeTerminator) {
+	for _, terminator := range terminators {
+		existing, _ := self.terminators.Get(terminator.terminatorId)
+		if existing == nil || existing == terminator { // make sure we're still the current terminator
+			terminator.setState(TerminatorStateDeleting, "deleting")
+			terminator.operationActive.Store(false)
+			self.deleteSet[terminator.terminatorId] = terminator
+		}
+	}
+}
+
+func (self *hostedServiceRegistry) RemoveTerminators(terminatorIds []string) error {
+	log := pfxlog.Logger()
+	request := &ctrl_pb.RemoveTerminatorsRequest{
+		TerminatorIds: terminatorIds,
+	}
+
+	ctrls := self.env.GetNetworkControllers()
+	ctrlCh := ctrls.AnyValidCtrlChannel()
+	if ctrlCh == nil {
+		return errors.New("no controller available")
+	}
+
+	responseMsg, err := protobufs.MarshalTyped(request).WithTimeout(ctrls.DefaultRequestTimeout()).SendForReply(ctrlCh)
+	if err != nil {
+		return fmt.Errorf("failed to send RemoveTerminatorsRequest message (%w)", err)
+	}
+
+	if responseMsg.ContentType != channel.ContentTypeResultType {
+		return fmt.Errorf("failure deleting terminators (unexpected response content type: %d)", responseMsg.ContentType)
+	}
+
+	result := channel.UnmarshalResult(responseMsg)
+	if result.Success {
+		return nil
+	}
+
+	if handler_common.WasRateLimited(responseMsg) {
+		log.Errorf("failure removing terminators (%v)", result.Message)
+		return apierror.NewTooManyUpdatesError()
+	}
+
+	return fmt.Errorf("failure deleting terminators (%s)", result.Message)
+}
+
+type queueEstablishTerminator struct {
+	terminator *edgeTerminator
+}
+
+func (self *queueEstablishTerminator) handle(registry *hostedServiceRegistry) {
+	registry.queueEstablishTerminatorSync(self.terminator)
+}
+
+type requeueRemoveTerminator struct {
 	terminator *edgeTerminator
 }
 
-func (self *establishTerminatorEvent) handle(registry *hostedServiceRegistry) {
-	registry.evaluateTerminator(self.terminator)
+func (self *requeueRemoveTerminator) handle(registry *hostedServiceRegistry) {
+	registry.requeueRemoveTerminatorSync(self.terminator)
 }
 
-type calculateRetry struct {
-	terminator  *edgeTerminator
-	queueFailed bool
+type queueRemoveTerminator struct {
+	terminator *edgeTerminator
+	reason     string
 }
 
-func (self *calculateRetry) handle(registry *hostedServiceRegistry) {
-	self.terminator.calculateRetry(self.queueFailed)
-	registry.retriesPending = true
+func (self *queueRemoveTerminator) handle(registry *hostedServiceRegistry) {
+	registry.queueRemoveTerminatorSync(self.terminator, self.reason)
 }
 
 func (self *hostedServiceRegistry) EstablishTerminator(terminator *edgeTerminator) {
-	self.Put(terminator.terminatorId.Load(), terminator)
-	self.queue(&establishTerminatorEvent{
-		terminator: terminator,
-	})
+	self.queueEstablishTerminatorAsync(terminator)
 }
 
 func (self *hostedServiceRegistry) queue(event terminatorEvent) {
@@ -128,155 +357,174 @@ func (self *hostedServiceRegistry) queue(event terminatorEvent) {
 	}
 }
 
-func (self *hostedServiceRegistry) scheduleRetry(terminator *edgeTerminator, queueFailed bool) {
-	terminator.establishActive.Store(false)
-	if terminator.state.CompareAndSwap(TerminatorStateEstablished, TerminatorStatePendingEstablishment) {
-		self.queue(&calculateRetry{
-			terminator:  terminator,
-			queueFailed: queueFailed,
-		})
+func (self *hostedServiceRegistry) queueWithTimeout(event terminatorEvent, timeout time.Duration) error {
+	select {
+	case self.events <- event:
+		return nil
+	case <-time.After(timeout):
+		return errors.New("timed out")
+	case <-self.env.GetCloseNotify():
+		return errors.New("closed")
 	}
 }
 
-func (self *hostedServiceRegistry) scanForRetries() {
-	self.services.Range(func(key, value any) bool {
-		terminator := value.(*edgeTerminator)
-		if terminator.state.Load() == TerminatorStatePendingEstablishment {
-			self.evaluateTerminator(terminator)
-		}
-		return true
-	})
+func (self *hostedServiceRegistry) queueEstablishTerminatorSync(terminator *edgeTerminator) {
+	if terminator.IsEstablishing() {
+		terminator.operationActive.Store(false)
+		self.establishSet[terminator.terminatorId] = terminator
+	}
 }
 
-func (self *hostedServiceRegistry) Put(hostId string, conn *edgeTerminator) {
-	self.services.Store(hostId, conn)
+func (self *hostedServiceRegistry) queueEstablishTerminatorAsync(terminator *edgeTerminator) {
+	self.queue(&queueEstablishTerminator{
+		terminator: terminator,
+	})
 }
 
-func (self *hostedServiceRegistry) Get(hostId string) (*edgeTerminator, bool) {
-	val, ok := self.services.Load(hostId)
-	if !ok {
-		return nil, false
-	}
-	ch, ok := val.(*edgeTerminator)
-	return ch, ok
+func (self *hostedServiceRegistry) queueRemoveTerminatorAsync(terminator *edgeTerminator, reason string) {
+	self.queue(&queueRemoveTerminator{
+		terminator: terminator,
+		reason:     reason,
+	})
 }
 
-func (self *hostedServiceRegistry) GetTerminatorForListener(listenerId string) *edgeTerminator {
-	var result *edgeTerminator
-	self.services.Range(func(key, value interface{}) bool {
-		terminator := value.(*edgeTerminator)
-		if terminator.listenerId == listenerId {
-			result = terminator
-			return false
-		}
-		return true
+func (self *hostedServiceRegistry) requeueRemoveTerminatorAsync(terminator *edgeTerminator) {
+	self.queue(&requeueRemoveTerminator{
+		terminator: terminator,
 	})
-	return result
 }
 
-func (self *hostedServiceRegistry) Delete(hostId string) {
-	self.services.Delete(hostId)
+func (self *hostedServiceRegistry) requeueRemoveTerminatorSync(terminator *edgeTerminator) {
+	existing, _ := self.terminators.Get(terminator.terminatorId)
+	if existing == nil || existing == terminator && terminator.state.Load() == TerminatorStateDeleting { // make sure we're still the current terminator
+		terminator.operationActive.Store(false)
+		self.deleteSet[terminator.terminatorId] = terminator
+	}
 }
 
-func (self *hostedServiceRegistry) cleanupServices(proxy *edgeClientConn) {
-	self.services.Range(func(key, value interface{}) bool {
-		terminator := value.(*edgeTerminator)
-		if terminator.edgeClientConn == proxy {
-			terminator.close(false, true, "channel closed") // don't notify, channel is already closed, we can't send messages
-			self.services.Delete(key)
-		}
-		return true
-	})
+func (self *hostedServiceRegistry) queueRemoveTerminatorSync(terminator *edgeTerminator, reason string) {
+	existing, _ := self.terminators.Get(terminator.terminatorId)
+	if existing == nil || existing == terminator { // make sure we're still the current terminator
+		self.queueRemoveTerminatorUnchecked(terminator, reason)
+	}
 }
 
-func (self *hostedServiceRegistry) cleanupDuplicates(newest *edgeTerminator) {
-	self.services.Range(func(key, value interface{}) bool {
-		terminator := value.(*edgeTerminator)
-		if terminator != newest && newest.token == terminator.token && newest.instance == terminator.instance {
-			terminator.close(false, true, "duplicate terminator") // don't notify, channel is already closed, we can't send messages
-			self.services.Delete(key)
-			pfxlog.Logger().WithField("routerId", terminator.edgeClientConn.listener.id.Token).
-				WithField("token", terminator.token).
-				WithField("instance", terminator.instance).
-				WithField("terminatorId", terminator.terminatorId.Load()).
-				WithField("duplicateOf", newest.terminatorId.Load()).
-				Info("duplicate removed")
-		}
-		return true
-	})
+func (self *hostedServiceRegistry) queueRemoveTerminatorUnchecked(terminator *edgeTerminator, reason string) {
+	terminator.setState(TerminatorStateDeleting, reason)
+	terminator.operationActive.Store(false)
+	self.deleteSet[terminator.terminatorId] = terminator
 }
 
-func (self *hostedServiceRegistry) unbindSession(connId uint32, sessionToken string, proxy *edgeClientConn) bool {
-	atLeastOneRemoved := false
-	self.services.Range(func(key, value interface{}) bool {
-		terminator := value.(*edgeTerminator)
-		if terminator.MsgChannel.Id() == connId && terminator.token == sessionToken && terminator.edgeClientConn == proxy {
-			terminator.close(false, true, "unbind successful") // don't notify, sdk asked us to unbind
-			self.services.Delete(key)
-			pfxlog.Logger().WithField("routerId", terminator.edgeClientConn.listener.id.Token).
-				WithField("token", sessionToken).
-				WithField("terminatorId", terminator.terminatorId.Load()).
-				Info("terminator removed")
-			atLeastOneRemoved = true
-		}
-		return true
+func (self *hostedServiceRegistry) markEstablished(terminator *edgeTerminator, reason string) {
+	self.queue(&markEstablishedEvent{
+		terminator: terminator,
+		reason:     reason,
 	})
-	return atLeastOneRemoved
 }
 
-func (self *hostedServiceRegistry) getRelatedTerminators(sessionToken string, proxy *edgeClientConn) []*edgeTerminator {
-	var result []*edgeTerminator
-	self.services.Range(func(key, value interface{}) bool {
-		terminator := value.(*edgeTerminator)
-		if terminator.token == sessionToken && terminator.edgeClientConn == proxy {
-			result = append(result, terminator)
+func (self *hostedServiceRegistry) scanForRetries() {
+	var retryList []*edgeTerminator
+
+	self.terminators.IterCb(func(_ string, terminator *edgeTerminator) {
+		state := terminator.state.Load()
+		if state.IsWorkRequired() && time.Since(terminator.lastAttempt) > 2*time.Minute {
+			retryList = append(retryList, terminator)
 		}
-		return true
 	})
-	return result
+
+	for _, terminator := range retryList {
+		state := terminator.state.Load()
+		if state == TerminatorStateEstablishing {
+			self.queueEstablishTerminatorSync(terminator)
+		} else if state == TerminatorStateDeleting {
+			self.requeueRemoveTerminatorSync(terminator)
+		}
+	}
 }
 
-func (self *hostedServiceRegistry) evaluateTerminator(terminator *edgeTerminator) {
-	log := logrus.
-		WithField("terminatorId", terminator.terminatorId.Load()).
-		WithField("state", terminator.state.Load()).
-		WithField("token", terminator.token)
+func (self *hostedServiceRegistry) PutV1(token string, terminator *edgeTerminator) {
+	self.terminators.Set(token, terminator)
+}
 
-	if terminator.edgeClientConn.ch.IsClosed() {
-		log.Info("terminator sdk channel closed, not trying to establish")
-		return
-	}
+func (self *hostedServiceRegistry) Put(terminator *edgeTerminator) {
+	self.terminators.Set(terminator.terminatorId, terminator)
+}
 
-	if terminator.terminatorId.Load() == "" {
-		log.Info("terminator has been closed, not trying to establish")
-		return
+func (self *hostedServiceRegistry) Get(terminatorId string) (*edgeTerminator, bool) {
+	return self.terminators.Get(terminatorId)
+}
+
+func (self *hostedServiceRegistry) Delete(terminatorId string) {
+	self.terminators.Remove(terminatorId)
+}
+
+func (self *hostedServiceRegistry) Remove(terminator *edgeTerminator, reason string) bool {
+	removed := self.terminators.RemoveCb(terminator.terminatorId, func(key string, v *edgeTerminator, exists bool) bool {
+		return v == terminator
+	})
+	if removed {
+		pfxlog.Logger().WithField("terminatorId", terminator.terminatorId).
+			WithField("reason", reason).
+			Info("terminator removed from router set")
 	}
+	return removed
+}
 
-	tryEstablish := terminator.state.Load() == TerminatorStatePendingEstablishment && terminator.nextAttempt.Before(time.Now())
+func (self *hostedServiceRegistry) cleanupServices(ch channel.Channel) {
+	self.queue(&channelClosedEvent{
+		ch: ch,
+	})
+}
 
-	if tryEstablish && terminator.establishActive.CompareAndSwap(false, true) {
-		if !terminator.state.CompareAndSwap(TerminatorStatePendingEstablishment, TerminatorStateEstablishing) {
-			log.Infof("terminator in state %s, not pending establishment, not queueing", terminator.state.Load())
-			return
+func (self *hostedServiceRegistry) cleanupDuplicates(newest *edgeTerminator) {
+	var toClose []*edgeTerminator
+	self.terminators.IterCb(func(_ string, terminator *edgeTerminator) {
+		if terminator != newest && newest.token == terminator.token && newest.instance == terminator.instance {
+			toClose = append(toClose, terminator)
 		}
+	})
 
-		log.Info("queuing terminator to send create")
+	for _, terminator := range toClose {
+		terminator.close(self, false, true, "duplicate terminator") // don't notify, channel is already closed, we can't send messages
+		pfxlog.Logger().WithField("routerId", terminator.edgeClientConn.listener.id.Token).
+			WithField("token", terminator.token).
+			WithField("instance", terminator.instance).
+			WithField("terminatorId", terminator.terminatorId).
+			WithField("duplicateOf", newest.terminatorId).
+			Info("duplicate removed")
+	}
+}
 
-		err := self.env.GetRateLimiterPool().QueueOrError(func() {
-			defer func() {
-				self.scheduleRetry(terminator, false)
-			}()
+func (self *hostedServiceRegistry) unbindSession(connId uint32, sessionToken string, proxy *edgeClientConn) bool {
+	var toClose []*edgeTerminator
+	self.terminators.IterCb(func(_ string, terminator *edgeTerminator) {
+		if terminator.MsgChannel.Id() == connId && terminator.token == sessionToken && terminator.edgeClientConn == proxy {
+			toClose = append(toClose, terminator)
+		}
+	})
 
-			if err := self.establishTerminator(terminator); err != nil {
-				log.WithError(err).Error("error establishing terminator")
-			}
-		})
+	atLeastOneRemoved := false
+	for _, terminator := range toClose {
+		terminator.close(self, false, true, "unbind successful") // don't notify, sdk asked us to unbind
+		pfxlog.Logger().WithField("routerId", terminator.edgeClientConn.listener.id.Token).
+			WithField("token", sessionToken).
+			WithField("connId", connId).
+			WithField("terminatorId", terminator.terminatorId).
+			Info("terminator removed")
+		atLeastOneRemoved = true
+	}
+	return atLeastOneRemoved
+}
 
-		if err != nil {
-			log.Info("rate limited: unable to queue to establish")
-			self.scheduleRetry(terminator, true)
+func (self *hostedServiceRegistry) getRelatedTerminators(connId uint32, sessionToken string, proxy *edgeClientConn) []*edgeTerminator {
+	var related []*edgeTerminator
+	self.terminators.IterCb(func(_ string, terminator *edgeTerminator) {
+		if terminator.MsgChannel.Id() == connId && terminator.token == sessionToken && terminator.edgeClientConn == proxy {
+			related = append(related, terminator)
 		}
-	}
+	})
+
+	return related
 }
 
 func (self *hostedServiceRegistry) establishTerminator(terminator *edgeTerminator) error {
@@ -284,16 +532,11 @@ func (self *hostedServiceRegistry) establishTerminator(terminator *edgeTerminato
 
 	log := pfxlog.Logger().
 		WithField("routerId", factory.env.GetRouterId().Token).
-		WithField("terminatorId", terminator.terminatorId.Load()).
+		WithField("terminatorId", terminator.terminatorId).
 		WithField("token", terminator.token)
 
-	terminatorId := terminator.terminatorId.Load()
-	if terminatorId == "" {
-		return fmt.Errorf("edge link is closed, stopping terminator creation for terminator %s", terminatorId)
-	}
-
 	request := &edge_ctrl_pb.CreateTerminatorV2Request{
-		Address:        terminatorId,
+		Address:        terminator.terminatorId,
 		SessionToken:   terminator.token,
 		Fingerprints:   terminator.edgeClientConn.fingerprints.Prints(),
 		PeerData:       terminator.hostData,
@@ -303,7 +546,6 @@ func (self *hostedServiceRegistry) establishTerminator(terminator *edgeTerminato
 		InstanceSecret: terminator.instanceSecret,
 	}
 
-	timeout := factory.ctrls.DefaultRequestTimeout()
 	ctrlCh := factory.ctrls.AnyCtrlChannel()
 	if ctrlCh == nil {
 		errStr := "no controller available, cannot create terminator"
@@ -313,10 +555,19 @@ func (self *hostedServiceRegistry) establishTerminator(terminator *edgeTerminato
 
 	log.Info("sending create terminator v2 request")
 
-	return protobufs.MarshalTyped(request).WithTimeout(timeout).SendAndWaitForWire(ctrlCh)
+	queued, err := ctrlCh.TrySend(protobufs.MarshalTyped(request).ToSendable())
+	if err != nil {
+		return err
+	}
+	if !queued {
+		return errors.New("channel too busy")
+	}
+	return nil
 }
 
 func (self *hostedServiceRegistry) HandleCreateTerminatorResponse(msg *channel.Message, _ channel.Channel) {
+	defer self.triggerEvaluates()
+
 	log := pfxlog.Logger().WithField("routerId", self.env.GetRouterId().Token)
 
 	response := &edge_ctrl_pb.CreateTerminatorV2Response{}
@@ -334,60 +585,380 @@ func (self *hostedServiceRegistry) HandleCreateTerminatorResponse(msg *channel.M
 		return
 	}
 
-	log = log.WithField("lifetime", time.Since(terminator.createTime))
+	log = log.WithField("lifetime", time.Since(terminator.createTime)).
+		WithField("connId", terminator.MsgChannel.Id())
 
 	if response.Result == edge_ctrl_pb.CreateTerminatorResult_FailedBusy {
 		log.Info("controller too busy to handle create terminator, retrying later")
+		if rateLimitCallback := terminator.GetAndClearRateLimitCallback(); rateLimitCallback != nil {
+			rateLimitCallback.Backoff()
+		}
+		self.queueEstablishTerminatorAsync(terminator)
 		return
 	}
 
 	if response.Result != edge_ctrl_pb.CreateTerminatorResult_Success {
+		if rateLimitCallback := terminator.GetAndClearRateLimitCallback(); rateLimitCallback != nil {
+			rateLimitCallback.Success()
+		}
+
+		terminator.operationActive.Store(false)
 		if terminator.establishCallback != nil {
-			terminator.establishCallback(false, response.Msg)
+			terminator.establishCallback(response.Result)
 		}
-		terminator.close(true, false, response.Msg)
+		terminator.close(self, true, false, response.Msg)
 		return
 	}
 
-	if terminator.state.CompareAndSwap(TerminatorStateEstablishing, TerminatorStateEstablished) {
-		log.Info("received terminator created notification")
-	} else {
-		log.Info("received additional terminator created notification")
-	}
+	self.markEstablished(terminator, "create notification received")
 
-	isValid := true
-	if terminator.postValidate {
-		if result, err := terminator.inspect(true); err != nil {
-			log.WithError(err).Error("error validating terminator after create")
-		} else if result.Type != edge.ConnTypeBind {
-			log.WithError(err).Error("terminator invalid in sdk after create, closed")
-			isValid = false
-		} else {
-			log.Info("terminator validated successfully")
-		}
-	}
+	// notify the sdk that the terminator was established
+	terminator.establishCallback(response.Result)
 
-	if isValid && terminator.establishCallback != nil {
-		terminator.establishCallback(true, "terminator established")
-	}
+	// we don't need to call inspect here, as the controller will be doing a post-create check
+	// to verify that everything is still in place
 }
 
 func (self *hostedServiceRegistry) HandleReconnect() {
 	var restablishList []*edgeTerminator
-	self.services.Range(func(key, value interface{}) bool {
-		terminator := value.(*edgeTerminator)
-		if terminator.state.CompareAndSwap(TerminatorStateEstablished, TerminatorStatePendingEstablishment) {
+	self.terminators.IterCb(func(_ string, terminator *edgeTerminator) {
+		if terminator.updateState(TerminatorStateEstablished, TerminatorStateEstablishing, "reconnecting") {
 			restablishList = append(restablishList, terminator)
 		}
-		return true
 	})
 
 	// wait for verify terminator events to come in
 	time.Sleep(10 * time.Second)
 
 	for _, terminator := range restablishList {
-		if terminator.state.Load() == TerminatorStatePendingEstablishment {
-			self.EstablishTerminator(terminator)
+		if terminator.state.Load() == TerminatorStateEstablishing {
+			self.queueEstablishTerminatorAsync(terminator)
 		}
 	}
 }
+
+func (self *hostedServiceRegistry) Inspect(timeout time.Duration) *inspect.SdkTerminatorInspectResult {
+	evt := &inspectTerminatorsEvent{
+		result: atomic.Pointer[[]*inspect.SdkTerminatorInspectDetail]{},
+		done:   make(chan struct{}),
+	}
+
+	// if we can't queue, grab the results in a non-thread-safe fashion
+	if err := self.queueWithTimeout(evt, timeout); err != nil {
+		evt.handle(self)
+	}
+
+	result := &inspect.SdkTerminatorInspectResult{}
+
+	var err error
+	result.Entries, err = evt.GetResults(timeout)
+	if err != nil {
+		result.Errors = append(result.Errors, err.Error())
+	}
+	return result
+}
+
+func (self *hostedServiceRegistry) checkForExistingListenerId(terminator *edgeTerminator) (listenerIdCheckResult, error) {
+	defaultResult := listenerIdCheckResult{
+		replaceExisting: false,
+		terminator:      terminator,
+	}
+
+	if terminator.listenerId == "" {
+		self.Put(terminator)
+		return defaultResult, nil
+	}
+
+	event := &findMatchingEvent{
+		terminator: terminator,
+		resultC:    make(chan listenerIdCheckResult, 1),
+	}
+	self.queue(event)
+
+	select {
+	case result := <-event.resultC:
+		return result, nil
+	case <-self.env.GetCloseNotify():
+		return defaultResult, errors.New("registry stopped")
+	case <-time.After(100 * time.Millisecond):
+
+		// if processing has already started, we need to wait for it to finish
+		// otherwise, we can return here
+		if event.cancelGate.CompareAndSwap(false, true) {
+			pfxlog.Logger().WithField("terminatorId", terminator.terminatorId).
+				WithField("listenerId", terminator.listenerId).
+				Info("unable to check for existing terminators with matching listenerId")
+
+			self.Put(terminator)
+			return defaultResult, nil
+		}
+
+		select {
+		case result := <-event.resultC:
+			return result, nil
+		case <-self.env.GetCloseNotify():
+			return defaultResult, errors.New("registry stopped")
+		}
+	}
+}
+
+func (self *hostedServiceRegistry) handleSdkReturnedInvalid(terminator *edgeTerminator, notifyCtrl bool) invalidTerminatorRemoveResult {
+	event := &sdkTerminatorInvalidEvent{
+		terminator: terminator,
+		resultC:    make(chan invalidTerminatorRemoveResult, 1),
+		notifyCtrl: notifyCtrl,
+	}
+	self.queue(event)
+
+	select {
+	case result := <-event.resultC:
+		return result
+	case <-time.After(100 * time.Millisecond):
+		if !self.terminators.Has(terminator.terminatorId) {
+			return invalidTerminatorRemoveResult{
+				existed: false,
+				removed: false,
+			}
+		}
+
+		return invalidTerminatorRemoveResult{
+			err: errors.New("operation timed out"),
+		}
+	}
+}
+
+type inspectTerminatorsEvent struct {
+	result atomic.Pointer[[]*inspect.SdkTerminatorInspectDetail]
+	done   chan struct{}
+}
+
+func (self *inspectTerminatorsEvent) handle(registry *hostedServiceRegistry) {
+	var result []*inspect.SdkTerminatorInspectDetail
+	registry.terminators.IterCb(func(key string, terminator *edgeTerminator) {
+		detail := &inspect.SdkTerminatorInspectDetail{
+			Key:             key,
+			Id:              terminator.terminatorId,
+			State:           terminator.state.Load().String(),
+			Token:           terminator.token,
+			ListenerId:      terminator.listenerId,
+			Instance:        terminator.instance,
+			Cost:            terminator.cost,
+			Precedence:      terminator.precedence.String(),
+			AssignIds:       terminator.assignIds,
+			V2:              terminator.v2,
+			SupportsInspect: terminator.supportsInspect,
+			OperationActive: terminator.operationActive.Load(),
+			CreateTime:      terminator.createTime.Format("2006-01-02 15:04:05"),
+			LastAttempt:     terminator.lastAttempt.Format("2006-01-02 15:04:05"),
+		}
+		result = append(result, detail)
+	})
+
+	self.result.Store(&result)
+	close(self.done)
+}
+
+func (self *inspectTerminatorsEvent) GetResults(timeout time.Duration) ([]*inspect.SdkTerminatorInspectDetail, error) {
+	select {
+	case <-self.done:
+		return *self.result.Load(), nil
+	case <-time.After(timeout):
+		return nil, errors.New("timed out waiting for result")
+	}
+}
+
+type channelClosedEvent struct {
+	ch channel.Channel
+}
+
+func (self *channelClosedEvent) handle(registry *hostedServiceRegistry) {
+	registry.terminators.IterCb(func(_ string, terminator *edgeTerminator) {
+		if terminator.MsgChannel.Channel == self.ch {
+			if terminator.v2 {
+				// we're iterating the map right now, so the terminator can't have changed
+				registry.queueRemoveTerminatorUnchecked(terminator, "channel closed")
+			} else {
+				// don't notify, channel is already closed, we can't send messages
+				go terminator.close(registry, false, true, "channel closed")
+			}
+		}
+	})
+}
+
+type listenerIdCheckResult struct {
+	replaceExisting bool
+	terminator      *edgeTerminator
+	previous        *edgeTerminator
+}
+
+type findMatchingEvent struct {
+	terminator *edgeTerminator
+	resultC    chan listenerIdCheckResult
+	cancelGate atomic.Bool
+}
+
+func (self *findMatchingEvent) handle(registry *hostedServiceRegistry) {
+	if !self.cancelGate.CompareAndSwap(false, true) {
+		return
+	}
+
+	// NOTE: We need to store the terminator in the map before we exit. If we do it later,
+	// another process for the same listener id might be in this code, and then we've got
+	// a race condition for which will end up in the map
+
+	var existingList []*edgeTerminator
+	registry.terminators.IterCb(func(_ string, terminator *edgeTerminator) {
+		if terminator.v2 && terminator.listenerId == self.terminator.listenerId {
+			existingList = append(existingList, terminator)
+		}
+	})
+
+	if len(existingList) == 0 {
+		registry.Put(self.terminator)
+		self.resultC <- listenerIdCheckResult{
+			terminator:      self.terminator,
+			replaceExisting: false,
+		}
+		return
+	}
+
+	log := pfxlog.ContextLogger(self.terminator.edgeClientConn.ch.Label()).
+		WithField("token", self.terminator.token).
+		WithField("routerId", self.terminator.edgeClientConn.listener.id.Token).
+		WithField("listenerId", self.terminator.listenerId).
+		WithField("connId", self.terminator.MsgChannel.Id()).
+		WithField("terminatorId", self.terminator.terminatorId)
+
+	for _, existing := range existingList {
+		matches := self.terminator.edgeClientConn == existing.edgeClientConn &&
+			self.terminator.token == existing.token &&
+			self.terminator.MsgChannel.Id() == existing.MsgChannel.Id() &&
+			existing.state.Load() != TerminatorStateDeleting
+
+		if matches {
+			log = log.WithField("existingTerminatorId", existing.terminatorId)
+			log.Info("duplicate bind request")
+			self.resultC <- listenerIdCheckResult{
+				terminator:      self.terminator,
+				replaceExisting: true,
+				previous:        existing,
+			}
+			return
+		}
+	}
+
+	for _, existing := range existingList {
+		if !existing.IsDeleting() {
+			self.terminator.replace(existing)
+			registry.Put(self.terminator)
+
+			// sometimes things happen close together. we need to try to notify replaced terminators
+			// that they're being closed in case they're the newer, still open connection
+			existing.close(registry, true, false, "found a newer terminator for listener id")
+			existing.setState(TerminatorStateDeleting, "newer terminator found for listener id")
+
+			log = log.WithField("existingTerminatorId", existing.terminatorId)
+			log.Info("taking over terminator from existing bind")
+
+			self.resultC <- listenerIdCheckResult{
+				terminator:      self.terminator,
+				replaceExisting: true,
+				previous:        existing,
+			}
+			return
+		}
+	}
+
+	// can't reuse terminator ID, if the existing terminator is being deleted
+	log.Info("existing terminator being deleted, need to establish new terminator")
+	registry.Put(self.terminator)
+	self.resultC <- listenerIdCheckResult{
+		terminator:      self.terminator,
+		replaceExisting: false,
+		previous:        existingList[0],
+	}
+}
+
+type invalidTerminatorRemoveResult struct {
+	err     error
+	existed bool
+	removed bool
+}
+
+type sdkTerminatorInvalidEvent struct {
+	terminator *edgeTerminator
+	resultC    chan invalidTerminatorRemoveResult
+	notifyCtrl bool
+}
+
+func (self *sdkTerminatorInvalidEvent) handle(registry *hostedServiceRegistry) {
+	self.resultC <- self.removeTerminator(registry)
+}
+
+func (self *sdkTerminatorInvalidEvent) removeTerminator(registry *hostedServiceRegistry) invalidTerminatorRemoveResult {
+	existing, _ := registry.terminators.Get(self.terminator.terminatorId)
+
+	if existing == nil {
+		return invalidTerminatorRemoveResult{
+			existed: false,
+			removed: false,
+		}
+	}
+
+	if existing != self.terminator {
+		return invalidTerminatorRemoveResult{
+			existed: true,
+			removed: false,
+		}
+	}
+
+	if self.notifyCtrl {
+		registry.queueRemoveTerminatorSync(self.terminator, "query to sdk indicated terminator is invalid")
+		return invalidTerminatorRemoveResult{
+			existed: true,
+			removed: true,
+		}
+
+	}
+
+	existed := false
+	removed := registry.terminators.RemoveCb(self.terminator.terminatorId, func(key string, v *edgeTerminator, exists bool) bool {
+		existed = exists
+		return v == self.terminator
+	})
+	if removed {
+		pfxlog.Logger().WithField("terminatorId", self.terminator.terminatorId).
+			WithField("reason", "query to sdk indicated terminator is invalid").
+			Info("terminator removed from router set")
+	}
+
+	return invalidTerminatorRemoveResult{
+		existed: existed,
+		removed: removed,
+	}
+}
+
+type markEstablishedEvent struct {
+	terminator *edgeTerminator
+	reason     string
+}
+
+func (self *markEstablishedEvent) handle(registry *hostedServiceRegistry) {
+	log := pfxlog.Logger().
+		WithField("routerId", registry.env.GetRouterId().Token).
+		WithField("terminatorId", self.terminator.terminatorId).
+		WithField("lifetime", time.Since(self.terminator.createTime)).
+		WithField("connId", self.terminator.MsgChannel.Id())
+
+	if rateLimitCallback := self.terminator.GetAndClearRateLimitCallback(); rateLimitCallback != nil {
+		rateLimitCallback.Success()
+	}
+
+	if !self.terminator.updateState(TerminatorStateEstablishing, TerminatorStateEstablished, self.reason) {
+		log.Info("received additional terminator created notification")
+	} else {
+		log.Info("terminator established")
+	}
+
+	self.terminator.operationActive.Store(false)
+}
diff --git a/router/xgress_edge/listener.go b/router/xgress_edge/listener.go
index 630e7d803..b9a678423 100644
--- a/router/xgress_edge/listener.go
+++ b/router/xgress_edge/listener.go
@@ -20,13 +20,13 @@ import (
 	"encoding/binary"
 	"fmt"
 	"github.com/openziti/ziti/common/ctrl_msg"
+	"github.com/openziti/ziti/controller/idgen"
 	"time"
 
 	"github.com/openziti/ziti/common/capabilities"
 	"github.com/openziti/ziti/common/cert"
 	fabricMetrics "github.com/openziti/ziti/common/metrics"
 	"github.com/openziti/ziti/common/pb/edge_ctrl_pb"
-	"github.com/openziti/ziti/controller/idgen"
 	"github.com/pkg/errors"
 	"google.golang.org/protobuf/proto"
 
@@ -109,10 +109,10 @@ type edgeClientConn struct {
 	idSeq        uint32
 }
 
-func (self *edgeClientConn) HandleClose(_ channel.Channel) {
+func (self *edgeClientConn) HandleClose(ch channel.Channel) {
 	log := pfxlog.ContextLogger(self.ch.Label())
 	log.Debugf("closing")
-	self.listener.factory.hostedServices.cleanupServices(self)
+	self.listener.factory.hostedServices.cleanupServices(ch)
 	self.msgMux.Close()
 }
 
@@ -305,7 +305,7 @@ func (self *edgeClientConn) processBindV1(req *channel.Message, ch channel.Chann
 
 	log.Debug("establishing listener")
 
-	messageSink := &edgeTerminator{
+	terminator := &edgeTerminator{
 		MsgChannel:     *edge.NewEdgeMsgChannel(self.ch, connId),
 		edgeClientConn: self,
 		token:          token,
@@ -314,11 +314,11 @@ func (self *edgeClientConn) processBindV1(req *channel.Message, ch channel.Chann
 	}
 
 	// need to remove session remove listener on close
-	messageSink.onClose = self.listener.factory.stateManager.AddEdgeSessionRemovedListener(token, func(token string) {
-		messageSink.close(true, true, "session ended")
+	terminator.onClose = self.listener.factory.stateManager.AddEdgeSessionRemovedListener(token, func(token string) {
+		terminator.close(self.listener.factory.hostedServices, true, true, "session ended")
 	})
 
-	self.listener.factory.hostedServices.Put(token, messageSink)
+	self.listener.factory.hostedServices.PutV1(token, terminator)
 
 	terminatorIdentity, _ := req.GetStringHeader(edge.TerminatorIdentityHeader)
 	var terminatorIdentitySecret []byte
@@ -340,18 +340,21 @@ func (self *edgeClientConn) processBindV1(req *channel.Message, ch channel.Chann
 	responseMsg, err := protobufs.MarshalTyped(request).WithTimeout(timeout).SendForReply(ctrlCh)
 	if err = xgress_common.CheckForFailureResult(responseMsg, err, edge_ctrl_pb.ContentType_CreateTerminatorResponseType); err != nil {
 		log.WithError(err).Warn("error creating terminator")
-		messageSink.close(false, false, "") // don't notify here, as we're notifying next line with a response
+		terminator.close(self.listener.factory.hostedServices, false, false, "") // don't notify here, as we're notifying next line with a response
 		self.sendStateClosedReply(err.Error(), req)
 		return
 	}
 
 	terminatorId := string(responseMsg.Body)
-	messageSink.terminatorId.Store(terminatorId)
-	log = log.WithField("terminatorId", terminatorIdentity)
+	terminator.lock.Lock()
+	terminator.terminatorId = terminatorId
+	terminator.lock.Unlock()
 
-	if messageSink.MsgChannel.IsClosed() {
+	log = log.WithField("terminatorId", terminatorId)
+
+	if terminator.MsgChannel.IsClosed() {
 		log.Warn("edge channel closed while setting up terminator. cleaning up terminator now")
-		messageSink.close(false, true, "edge channel closed")
+		terminator.close(self.listener.factory.hostedServices, false, true, "edge channel closed")
 		return
 	}
 
@@ -370,43 +373,26 @@ func (self *edgeClientConn) processBindV2(req *channel.Message, ch channel.Chann
 		WithFields(edge.GetLoggerFields(req)).
 		WithField("routerId", self.listener.id.Token)
 
+	if self.listener.factory.stateManager.WasSessionRecentlyRemoved(token) {
+		log.Info("invalid session, not establishing terminator")
+		self.sendStateClosedReply("invalid session", req)
+		return
+	}
+
 	connId, found := req.GetUint32Header(edge.ConnIdHeader)
 	if !found {
 		pfxlog.Logger().Errorf("connId not set. unable to process bind message")
 		return
 	}
 
-	var terminatorId string
+	terminatorId := idgen.NewUUIDString()
+	log = log.WithField("bindConnId", connId).WithField("terminatorId", terminatorId)
 
 	listenerId, _ := req.GetStringHeader(edge.ListenerId)
 	if listenerId != "" {
 		log = log.WithField("listenerId", listenerId)
-		if terminator := self.listener.factory.hostedServices.GetTerminatorForListener(listenerId); terminator != nil {
-			terminatorId = terminator.terminatorId.Load()
-			log = log.WithField("terminatorId", terminatorId)
-
-			// everything is the same, we can reuse the terminator
-			if terminator.edgeClientConn == self && terminator.token == token {
-				log.Info("duplicate create terminator request")
-				self.sendStateConnectedReply(req, nil)
-				return
-			}
-
-			if terminator.terminatorId.CompareAndSwap(terminatorId, "") {
-				log.Info("replacing existing terminator")
-				self.listener.factory.hostedServices.Delete(terminatorId)
-			} else {
-				terminatorId = idgen.NewUUIDString()
-				log.Infof("unable to replace existing terminator, as it's being shut down, creating new one with id %s", terminatorId)
-			}
-		}
 	}
 
-	if terminatorId == "" {
-		terminatorId = idgen.NewUUIDString()
-	}
-
-	log = log.WithField("bindConnId", connId).WithField("terminatorId", terminatorId)
 	terminatorInstance, _ := req.GetStringHeader(edge.TerminatorIdentityHeader)
 
 	assignIds, _ := req.GetBoolHeader(edge.RouterProvidedConnId)
@@ -437,63 +423,83 @@ func (self *edgeClientConn) processBindV2(req *channel.Message, ch channel.Chann
 		hostData[edge.PublicKeyHeader] = pubKey
 	}
 
-	postValidate, _ := req.GetBoolHeader(edge.SupportsInspectHeader)
+	supportsInspect, _ := req.GetBoolHeader(edge.SupportsInspectHeader)
 	notifyEstablished, _ := req.GetBoolHeader(edge.SupportsBindSuccessHeader)
 
 	terminator := &edgeTerminator{
-		MsgChannel:     *edge.NewEdgeMsgChannel(self.ch, connId),
-		edgeClientConn: self,
-		token:          token,
-		cost:           cost,
-		precedence:     precedence,
-		instance:       terminatorInstance,
-		instanceSecret: terminatorInstanceSecret,
-		hostData:       hostData,
-		assignIds:      assignIds,
-		v2:             true,
-		postValidate:   postValidate,
-		createTime:     time.Now(),
+		terminatorId:    terminatorId,
+		MsgChannel:      *edge.NewEdgeMsgChannel(self.ch, connId),
+		edgeClientConn:  self,
+		token:           token,
+		listenerId:      listenerId,
+		cost:            cost,
+		precedence:      precedence,
+		instance:        terminatorInstance,
+		instanceSecret:  terminatorInstanceSecret,
+		hostData:        hostData,
+		assignIds:       assignIds,
+		v2:              true,
+		supportsInspect: supportsInspect,
+		createTime:      time.Now(),
+	}
+
+	terminator.state.Store(TerminatorStateEstablishing)
+
+	checkResult, err := self.listener.factory.hostedServices.checkForExistingListenerId(terminator)
+	if err != nil {
+		log.WithError(err).Error("error, cancelling processing")
+		return
 	}
 
-	terminator.establishCallback = func(ok bool, msg string) {
-		if ok {
-			self.sendStateConnectedReply(req, nil)
-
-			if notifyEstablished {
-				notifyMsg := channel.NewMessage(edge.ContentTypeBindSuccess, nil)
-				notifyMsg.PutUint32Header(edge.ConnIdHeader, terminator.MsgChannel.Id())
+	terminator = checkResult.terminator
+	if terminator.state.Load() == TerminatorStateDeleting {
+		return
+	}
 
-				if err := notifyMsg.WithTimeout(time.Second * 30).Send(terminator.MsgChannel.Channel); err != nil {
-					log.WithError(err).Error("failed to send bind success")
-				}
-			}
-		} else {
-			self.sendStateClosedReply(msg, req)
-		}
+	if checkResult.previous == nil || checkResult.previous.token != token {
+		// need to remove session remove listener on close
+		terminator.onClose = self.listener.factory.stateManager.AddEdgeSessionRemovedListener(token, func(token string) {
+			terminator.close(self.listener.factory.hostedServices, true, true, "session ended")
+		})
 	}
 
-	terminator.terminatorId.Store(terminatorId)
-	terminator.state.Store(TerminatorStatePendingEstablishment)
+	terminator.establishCallback = func(result edge_ctrl_pb.CreateTerminatorResult) {
+		if result == edge_ctrl_pb.CreateTerminatorResult_Success && notifyEstablished {
+			notifyMsg := channel.NewMessage(edge.ContentTypeBindSuccess, nil)
+			notifyMsg.PutUint32Header(edge.ConnIdHeader, terminator.MsgChannel.Id())
 
-	if self.listener.factory.stateManager.WasSessionRecentlyRemoved(token) {
-		log.Info("invalid session, not establishing terminator")
-		terminator.establishCallback(false, "invalid session")
-		return
+			if err := notifyMsg.WithTimeout(time.Second * 30).Send(terminator.MsgChannel.Channel); err != nil {
+				log.WithError(err).Error("failed to send bind success")
+			} else {
+				log.Info("sdk notified of terminator creation")
+			}
+		} else if result == edge_ctrl_pb.CreateTerminatorResult_FailedInvalidSession {
+			// TODO: notify of invalid session. Currently handling this using the recently removed sessions
+			//       LRU cache in state manager
+			log.Trace("invalid session")
+		}
 	}
 
-	// need to remove session remove listener on close
-	terminator.onClose = self.listener.factory.stateManager.AddEdgeSessionRemovedListener(token, func(token string) {
-		terminator.close(true, true, "session ended")
-	})
-
-	log.Info("establishing terminator")
+	self.sendStateConnectedReply(req, nil)
 
-	self.listener.factory.hostedServices.EstablishTerminator(terminator)
-	if listenerId == "" {
-		// only removed dupes with a scan if we don't have an sdk provided key
-		self.listener.factory.hostedServices.cleanupDuplicates(terminator)
+	if checkResult.replaceExisting {
+		log.Info("sending replacement terminator success to sdk")
+		terminator.establishCallback(edge_ctrl_pb.CreateTerminatorResult_Success)
+		if terminator.supportsInspect {
+			go func() {
+				if _, err := terminator.inspect(self.listener.factory.hostedServices, true, true); err != nil {
+					log.WithError(err).Info("failed to check sdk side of terminator after replace")
+				}
+			}()
+		}
+	} else {
+		log.Info("establishing terminator")
+		self.listener.factory.hostedServices.EstablishTerminator(terminator)
+		if listenerId == "" {
+			// only removed dupes with a scan if we don't have an sdk provided key
+			self.listener.factory.hostedServices.cleanupDuplicates(terminator)
+		}
 	}
-
 }
 
 func (self *edgeClientConn) processUnbind(req *channel.Message, _ channel.Channel) {
@@ -524,8 +530,10 @@ func (self *edgeClientConn) removeTerminator(ctrlCh channel.Channel, token, term
 func (self *edgeClientConn) processUpdateBind(req *channel.Message, ch channel.Channel) {
 	token := string(req.Body)
 
+	connId, _ := req.GetUint32Header(edge.ConnIdHeader)
+
 	log := pfxlog.ContextLogger(ch.Label()).WithField("token", token).WithFields(edge.GetLoggerFields(req))
-	terminators := self.listener.factory.hostedServices.getRelatedTerminators(token, self)
+	terminators := self.listener.factory.hostedServices.getRelatedTerminators(connId, token, self)
 
 	if len(terminators) == 0 {
 		log.Error("failed to update bind, no listener found")
@@ -542,7 +550,7 @@ func (self *edgeClientConn) processUpdateBind(req *channel.Message, ch channel.C
 		request := &edge_ctrl_pb.UpdateTerminatorRequest{
 			SessionToken: token,
 			Fingerprints: self.fingerprints.Prints(),
-			TerminatorId: terminator.terminatorId.Load(),
+			TerminatorId: terminator.terminatorId,
 		}
 
 		if costVal, hasCost := req.GetUint16Header(edge.CostHeader); hasCost {
@@ -561,7 +569,7 @@ func (self *edgeClientConn) processUpdateBind(req *channel.Message, ch channel.C
 			}
 		}
 
-		log = log.WithField("terminator", terminator.terminatorId.Load()).
+		log = log.WithField("terminator", terminator.terminatorId).
 			WithField("precedence", request.Precedence).
 			WithField("cost", request.Cost).
 			WithField("updatingPrecedence", request.UpdatePrecedence).
@@ -601,11 +609,11 @@ func (self *edgeClientConn) processHealthEvent(req *channel.Message, ch channel.
 	request := &edge_ctrl_pb.HealthEventRequest{
 		SessionToken: token,
 		Fingerprints: self.fingerprints.Prints(),
-		TerminatorId: terminator.terminatorId.Load(),
+		TerminatorId: terminator.terminatorId,
 		CheckPassed:  checkPassed,
 	}
 
-	log = log.WithField("terminator", terminator.terminatorId.Load()).WithField("checkPassed", checkPassed)
+	log = log.WithField("terminator", terminator.terminatorId).WithField("checkPassed", checkPassed)
 	log.Debug("sending health event")
 
 	if err := protobufs.MarshalTyped(request).Send(ctrlCh); err != nil {
diff --git a/router/xgress_edge/terminator_heap.go b/router/xgress_edge/terminator_heap.go
deleted file mode 100644
index da0e72b00..000000000
--- a/router/xgress_edge/terminator_heap.go
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
-	(c) Copyright NetFoundry Inc.
-
-	Licensed under the Apache License, Version 2.0 (the "License");
-	you may not use this file except in compliance with the License.
-	You may obtain a copy of the License at
-
-	https://www.apache.org/licenses/LICENSE-2.0
-
-	Unless required by applicable law or agreed to in writing, software
-	distributed under the License is distributed on an "AS IS" BASIS,
-	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-	See the License for the specific language governing permissions and
-	limitations under the License.
-*/
-
-package xgress_edge
-
-type terminatorHeap []*edgeTerminator
-
-func (self terminatorHeap) Len() int {
-	return len(self)
-}
-
-func (self terminatorHeap) Less(i, j int) bool {
-	return self[i].nextAttempt.Before(self[j].nextAttempt)
-}
-
-func (self terminatorHeap) Swap(i, j int) {
-	tmp := self[i]
-	self[i] = self[j]
-	self[j] = tmp
-}
-
-func (self *terminatorHeap) Push(x any) {
-	*self = append(*self, x.(*edgeTerminator))
-}
-
-func (self *terminatorHeap) Pop() any {
-	old := *self
-	n := len(old)
-	pm := old[n-1]
-	*self = old[0 : n-1]
-	return pm
-}
diff --git a/version b/version
index 00d0c14da..94b357edf 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-0.32
+0.33
diff --git a/ziti/cmd/cmd.go b/ziti/cmd/cmd.go
index 733d373c7..64fd9fa1e 100644
--- a/ziti/cmd/cmd.go
+++ b/ziti/cmd/cmd.go
@@ -131,7 +131,7 @@ func NewCmdRoot(in io.Reader, out, err io.Writer, cmd *cobra.Command) *cobra.Com
 	agentCommands := agentcli.NewAgentCmd(p)
 	pkiCommands := pki.NewCmdPKI(out, err)
 	fabricCommand := fabric.NewFabricCmd(p)
-	edgeCommand := edge.NewCmdEdge(out, err)
+	edgeCommand := edge.NewCmdEdge(out, err, p)
 	demoCmd := demo.NewDemoCmd(p)
 
 	opsCommands := &cobra.Command{
diff --git a/ziti/cmd/database/add_debug_admin.go b/ziti/cmd/database/add_debug_admin.go
index c8b40b30b..020946241 100644
--- a/ziti/cmd/database/add_debug_admin.go
+++ b/ziti/cmd/database/add_debug_admin.go
@@ -69,7 +69,7 @@ func (action *addDebugAdminAction) run(dbFile, username, password string) {
 	boltDb, err := db.Open(dbFile)
 	action.noError(err)
 
-	fabricStores, err := db.InitStores(boltDb)
+	fabricStores, err := db.InitStores(boltDb, command.NoOpRateLimiter{})
 	action.noError(err)
 
 	dispatcher := &command.LocalDispatcher{
@@ -83,7 +83,7 @@ func (action *addDebugAdminAction) run(dbFile, username, password string) {
 		managers: controllers,
 	}
 
-	stores, err := db.InitStores(boltDb)
+	stores, err := db.InitStores(boltDb, command.NoOpRateLimiter{})
 	action.noError(err)
 
 	id := "debug-admin"
diff --git a/ziti/cmd/edge/root.go b/ziti/cmd/edge/root.go
index 3538ffbda..a8e16155a 100644
--- a/ziti/cmd/edge/root.go
+++ b/ziti/cmd/edge/root.go
@@ -19,6 +19,7 @@ package edge
 import (
 	"context"
 	"github.com/openziti/ziti/ziti/cmd/common"
+	cmdhelper "github.com/openziti/ziti/ziti/cmd/helpers"
 	"github.com/openziti/ziti/ziti/util"
 	"io"
 
@@ -29,13 +30,9 @@ import (
 var ExtraEdgeCommands []func(p common.OptionsProvider) *cobra.Command
 
 // NewCmdEdge creates a command object for the "controller" command
-func NewCmdEdge(out io.Writer, errOut io.Writer) *cobra.Command {
+func NewCmdEdge(out io.Writer, errOut io.Writer, p common.OptionsProvider) *cobra.Command {
 	cmd := util.NewEmptyParentCmd("edge", "Manage the Edge components of a Ziti network using the Ziti Edge REST API")
-	populateEdgeCommands(out, errOut, cmd)
-	return cmd
-}
 
-func populateEdgeCommands(out io.Writer, errOut io.Writer, cmd *cobra.Command) *cobra.Command {
 	cmd.AddCommand(newCreateCmd(out, errOut))
 	cmd.AddCommand(newDeleteCmd(out, errOut))
 	cmd.AddCommand(NewLoginCmd(out, errOut))
@@ -52,12 +49,25 @@ func populateEdgeCommands(out io.Writer, errOut io.Writer, cmd *cobra.Command) *
 	cmd.AddCommand(newShowCmd(out, errOut))
 	cmd.AddCommand(newReEnrollCmd(out, errOut))
 	cmd.AddCommand(NewQuickStartCmd(out, errOut, context.Background()))
-
-	p := common.NewOptionsProvider(out, errOut)
+	cmd.AddCommand(newValidateCommand(p))
 	cmd.AddCommand(enrollment.NewEnrollCommand(p))
+
 	for _, cmdF := range ExtraEdgeCommands {
 		cmd.AddCommand(cmdF(p))
 	}
 
 	return cmd
 }
+
+func newValidateCommand(p common.OptionsProvider) *cobra.Command {
+	validateCmd := &cobra.Command{
+		Use:   "validate",
+		Short: "validate model data",
+		Run: func(cmd *cobra.Command, args []string) {
+			cmdhelper.CheckErr(cmd.Help())
+		},
+	}
+
+	validateCmd.AddCommand(NewValidateServiceHostingCmd(p))
+	return validateCmd
+}
diff --git a/ziti/cmd/edge/validate_service_hosting.go b/ziti/cmd/edge/validate_service_hosting.go
new file mode 100644
index 000000000..44a62574a
--- /dev/null
+++ b/ziti/cmd/edge/validate_service_hosting.go
@@ -0,0 +1,131 @@
+/*
+	Copyright NetFoundry Inc.
+
+	Licensed under the Apache License, Version 2.0 (the "License");
+	you may not use this file except in compliance with the License.
+	You may obtain a copy of the License at
+
+	https://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+	distributed under the License is distributed on an "AS IS" BASIS,
+	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+	See the License for the specific language governing permissions and
+	limitations under the License.
+*/
+
+package edge
+
+import (
+	"fmt"
+	"github.com/openziti/edge-api/rest_management_api_client/service"
+	"github.com/openziti/ziti/controller/rest_client/terminator"
+	"github.com/openziti/ziti/controller/rest_model"
+	"github.com/openziti/ziti/ziti/cmd/api"
+	"github.com/openziti/ziti/ziti/cmd/common"
+	"github.com/openziti/ziti/ziti/util"
+	"github.com/spf13/cobra"
+)
+
+type validateServiceHostingAction struct {
+	api.Options
+	filter string
+}
+
+func NewValidateServiceHostingCmd(p common.OptionsProvider) *cobra.Command {
+	action := validateServiceHostingAction{
+		Options: api.Options{
+			CommonOptions: p(),
+		},
+	}
+
+	validateServiceHostingCmd := &cobra.Command{
+		Use:     "service-hosting",
+		Short:   "Validate service hosting by comparing what is allowed to host services with what actually is hosting",
+		Example: "ziti fabric validate service-hosting --filter 'name=\"test\"' --show-only-invalid",
+		Args:    cobra.ExactArgs(0),
+		RunE:    action.validateServiceHosting,
+	}
+
+	action.AddCommonFlags(validateServiceHostingCmd)
+	validateServiceHostingCmd.Flags().StringVar(&action.filter, "filter", "sort by name limit none", "Specify which services to validate")
+	return validateServiceHostingCmd
+}
+
+func (self *validateServiceHostingAction) validateServiceHosting(cmd *cobra.Command, _ []string) error {
+	client, err := util.NewEdgeManagementClient(&self.Options)
+
+	if err != nil {
+		return err
+	}
+
+	fabricClient, err := util.NewFabricManagementClient(&self.Options)
+	if err != nil {
+		return err
+	}
+
+	context, cancelContext := self.Options.TimeoutContext()
+	defer cancelContext()
+
+	result, err := client.Service.ListServices(&service.ListServicesParams{
+		Filter:  &self.filter,
+		Context: context,
+	}, nil)
+
+	if err != nil {
+		return util.WrapIfApiError(err)
+	}
+
+	policyType := "bind"
+
+	limitNoneFilter := "limit none"
+	terminatorResult, err := fabricClient.Terminator.ListTerminators(&terminator.ListTerminatorsParams{
+		Filter:  &limitNoneFilter,
+		Limit:   nil,
+		Offset:  nil,
+		Context: context,
+	})
+	if err != nil {
+		return err
+	}
+
+	terminatorsBySvcAndHost := map[string]map[string][]*rest_model.TerminatorDetail{}
+	for _, detail := range terminatorResult.Payload.Data {
+		byHost, ok := terminatorsBySvcAndHost[*detail.ServiceID]
+		if !ok {
+			byHost = map[string][]*rest_model.TerminatorDetail{}
+			terminatorsBySvcAndHost[*detail.ServiceID] = byHost
+		}
+		byHost[*detail.HostID] = append(byHost[*detail.HostID], detail)
+	}
+
+	for _, svc := range result.Payload.Data {
+		identitiesResult, err := client.Service.ListServiceIdentities(&service.ListServiceIdentitiesParams{
+			ID:         *svc.ID,
+			PolicyType: &policyType,
+			Filter:     &limitNoneFilter,
+			Context:    context,
+		}, nil)
+		if err != nil {
+			return err
+		}
+
+		identities := identitiesResult.Payload.Data
+		if len(identities) == 0 {
+			fmt.Printf("service '%s' is not hostable by any identities\n", *svc.Name)
+		}
+
+		for _, identity := range identities {
+			var list []*rest_model.TerminatorDetail
+			if byHost, ok := terminatorsBySvcAndHost[*svc.ID]; ok {
+				list = byHost[*identity.ID]
+			}
+			fmt.Printf("service %s (%s) hosted by %s (%s) with %d terminators\n",
+				*svc.Name, *svc.ID,
+				*identity.Name, *identity.ID,
+				len(list))
+		}
+	}
+
+	return nil
+}
diff --git a/ziti/cmd/fabric/inspect.go b/ziti/cmd/fabric/inspect.go
index fae970d50..4ab0dda36 100644
--- a/ziti/cmd/fabric/inspect.go
+++ b/ziti/cmd/fabric/inspect.go
@@ -5,10 +5,10 @@ import (
 	"encoding/json"
 	"fmt"
 	"github.com/fatih/color"
-	"github.com/openziti/ziti/controller/rest_client/inspect"
-	"github.com/openziti/ziti/controller/rest_model"
 	"github.com/openziti/foundation/v2/errorz"
 	"github.com/openziti/foundation/v2/stringz"
+	"github.com/openziti/ziti/controller/rest_client/inspect"
+	"github.com/openziti/ziti/controller/rest_model"
 	"github.com/openziti/ziti/ziti/cmd/api"
 	"github.com/openziti/ziti/ziti/cmd/common"
 	"github.com/openziti/ziti/ziti/util"
@@ -29,7 +29,10 @@ func newInspectCmd(p common.OptionsProvider) *cobra.Command {
 	cmd.AddCommand(action.newInspectSubCmd(p, "config", "gets configuration from the requested nodes"))
 	cmd.AddCommand(action.newInspectSubCmd(p, "cluster-config", "gets a subset of cluster configuration from the requested nodes"))
 	cmd.AddCommand(action.newInspectSubCmd(p, "connected-routers", "gets information about which routers are connected to which controllers"))
+	cmd.AddCommand(action.newInspectSubCmd(p, "connected-peers", "gets information about which controllers are connected to which other controllers in the cluster"))
 	cmd.AddCommand(action.newInspectSubCmd(p, "links", "gets information from routers about their view of links"))
+	cmd.AddCommand(action.newInspectSubCmd(p, "sdk-terminators", "gets information from routers about their view of sdk terminators"))
+	cmd.AddCommand(action.newInspectSubCmd(p, "router-messaging", "gets information about pending router peer updates and terminator validations"))
 
 	inspectCircuitsAction := &InspectCircuitsAction{InspectAction: *newInspectAction(p)}
 	cmd.AddCommand(inspectCircuitsAction.newCobraCmd())
diff --git a/ziti/cmd/fabric/root.go b/ziti/cmd/fabric/root.go
index d4da670eb..8078ad537 100644
--- a/ziti/cmd/fabric/root.go
+++ b/ziti/cmd/fabric/root.go
@@ -111,6 +111,7 @@ func newValidateCommand(p common.OptionsProvider) *cobra.Command {
 
 	validateCmd.AddCommand(NewValidateTerminatorsCmd(p))
 	validateCmd.AddCommand(NewValidateRouterLinksCmd(p))
+	validateCmd.AddCommand(NewValidateRouterSdkTerminatorsCmd(p))
 	return validateCmd
 }
 
diff --git a/ziti/cmd/fabric/validate_router_sdk_terminators.go b/ziti/cmd/fabric/validate_router_sdk_terminators.go
new file mode 100644
index 000000000..2b6c489cb
--- /dev/null
+++ b/ziti/cmd/fabric/validate_router_sdk_terminators.go
@@ -0,0 +1,158 @@
+/*
+	Copyright NetFoundry Inc.
+
+	Licensed under the Apache License, Version 2.0 (the "License");
+	you may not use this file except in compliance with the License.
+	You may obtain a copy of the License at
+
+	https://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+	distributed under the License is distributed on an "AS IS" BASIS,
+	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+	See the License for the specific language governing permissions and
+	limitations under the License.
+*/
+
+package fabric
+
+import (
+	"fmt"
+	"github.com/michaelquigley/pfxlog"
+	"github.com/openziti/channel/v2"
+	"github.com/openziti/channel/v2/protobufs"
+	"github.com/openziti/ziti/common/pb/mgmt_pb"
+	"github.com/openziti/ziti/ziti/cmd/api"
+	"github.com/openziti/ziti/ziti/cmd/common"
+	"github.com/spf13/cobra"
+	"google.golang.org/protobuf/proto"
+	"os"
+	"time"
+)
+
+type validateRouterSdkTerminatorsAction struct {
+	api.Options
+	includeValidSdkTerminators bool
+	includeValidRouters        bool
+
+	eventNotify chan *mgmt_pb.RouterSdkTerminatorsDetails
+}
+
+func NewValidateRouterSdkTerminatorsCmd(p common.OptionsProvider) *cobra.Command {
+	action := validateRouterSdkTerminatorsAction{
+		Options: api.Options{
+			CommonOptions: p(),
+		},
+	}
+
+	validateSdkTerminatorsCmd := &cobra.Command{
+		Use:     "router-sdk-terminators <router filter>",
+		Short:   "Validate router sdk terminators",
+		Example: "ziti fabric validate router-sdk-terminators --filter 'name=\"my-router\"' --include-valid",
+		Args:    cobra.MaximumNArgs(1),
+		RunE:    action.validateRouterSdkTerminators,
+	}
+
+	action.AddCommonFlags(validateSdkTerminatorsCmd)
+	validateSdkTerminatorsCmd.Flags().BoolVar(&action.includeValidSdkTerminators, "include-valid-terminators", false, "Don't hide results for valid SdkTerminators")
+	validateSdkTerminatorsCmd.Flags().BoolVar(&action.includeValidRouters, "include-valid-routers", false, "Don't hide results for valid routers")
+	return validateSdkTerminatorsCmd
+}
+
+func (self *validateRouterSdkTerminatorsAction) validateRouterSdkTerminators(_ *cobra.Command, args []string) error {
+	closeNotify := make(chan struct{})
+	self.eventNotify = make(chan *mgmt_pb.RouterSdkTerminatorsDetails, 1)
+
+	bindHandler := func(binding channel.Binding) error {
+		binding.AddReceiveHandler(int32(mgmt_pb.ContentType_ValidateRouterSdkTerminatorsResultType), self)
+		binding.AddCloseHandler(channel.CloseHandlerF(func(ch channel.Channel) {
+			close(closeNotify)
+		}))
+		return nil
+	}
+
+	ch, err := api.NewWsMgmtChannel(channel.BindHandlerF(bindHandler))
+	if err != nil {
+		return err
+	}
+
+	filter := ""
+	if len(args) > 0 {
+		filter = args[0]
+	}
+
+	request := &mgmt_pb.ValidateRouterSdkTerminatorsRequest{
+		Filter: filter,
+	}
+
+	responseMsg, err := protobufs.MarshalTyped(request).WithTimeout(time.Duration(self.Timeout) * time.Second).SendForReply(ch)
+
+	response := &mgmt_pb.ValidateRouterSdkTerminatorsResponse{}
+	if err = protobufs.TypedResponse(response).Unmarshall(responseMsg, err); err != nil {
+		return err
+	}
+
+	if !response.Success {
+		return fmt.Errorf("failed to start sdk terminator validation: %s", response.Message)
+	}
+
+	fmt.Printf("started validation of %v routers\n", response.RouterCount)
+
+	expected := response.RouterCount
+
+	errCount := 0
+	for expected > 0 {
+		select {
+		case <-closeNotify:
+			fmt.Printf("channel closed, exiting")
+			return nil
+		case routerDetail := <-self.eventNotify:
+			result := "validation successful"
+			if !routerDetail.ValidateSuccess {
+				result = fmt.Sprintf("error: unable to validate (%s)", routerDetail.Message)
+				errCount++
+			}
+
+			routerHeaderDone := false
+			outputRouterHeader := func() {
+				fmt.Printf("routerId: %s, routerName: %v, sdk-terminators: %v, %s\n",
+					routerDetail.RouterId, routerDetail.RouterName, len(routerDetail.Details), result)
+				routerHeaderDone = true
+			}
+
+			if self.includeValidRouters || routerDetail.Message != "" {
+				outputRouterHeader()
+			}
+
+			for _, detail := range routerDetail.Details {
+				if self.includeValidSdkTerminators || !detail.IsValid {
+					if !routerHeaderDone {
+						outputRouterHeader()
+					}
+					fmt.Printf("\tid: %s, ctrlState: %v, routerState: %s, created: %s, lastAttempt: %s, reqOutstanding: %v\n",
+						detail.TerminatorId, detail.CtrlState, detail.RouterState,
+						detail.CreateTime, detail.LastAttempt, detail.OperaationActive)
+				}
+				if !detail.IsValid {
+					errCount++
+				}
+			}
+			expected--
+		}
+	}
+	fmt.Printf("%v errors found\n", errCount)
+	if errCount > 0 {
+		os.Exit(1)
+	}
+	return nil
+}
+
+func (self *validateRouterSdkTerminatorsAction) HandleReceive(msg *channel.Message, _ channel.Channel) {
+	detail := &mgmt_pb.RouterSdkTerminatorsDetails{}
+	if err := proto.Unmarshal(msg.Body, detail); err != nil {
+		pfxlog.Logger().WithError(err).Error("unable to unmarshal router sdk terminator details")
+		return
+	}
+
+	self.eventNotify <- detail
+}
diff --git a/ziti/cmd/fabric/validate_terminators.go b/ziti/cmd/fabric/validate_terminators.go
index 59c2c3d64..067d74cee 100644
--- a/ziti/cmd/fabric/validate_terminators.go
+++ b/ziti/cmd/fabric/validate_terminators.go
@@ -31,9 +31,9 @@ import (
 
 type validateTerminatorsAction struct {
 	api.Options
-	filter          string
-	fixInvalid      bool
-	showOnlyInvalid bool
+	filter       string
+	fixInvalid   bool
+	includeValid bool
 
 	eventNotify chan *mgmt_pb.TerminatorDetail
 }
@@ -55,7 +55,7 @@ func NewValidateTerminatorsCmd(p common.OptionsProvider) *cobra.Command {
 
 	action.AddCommonFlags(validateTerminatorsCmd)
 	validateTerminatorsCmd.Flags().BoolVar(&action.fixInvalid, "fix-invalid", false, "Fix invalid terminators. Usually this means deleting them.")
-	validateTerminatorsCmd.Flags().BoolVar(&action.showOnlyInvalid, "show-only-invalid", false, "Hide results for valid terminators")
+	validateTerminatorsCmd.Flags().BoolVar(&action.includeValid, "include-valid", false, "Show results for valid terminators as well")
 	validateTerminatorsCmd.Flags().StringVar(&action.filter, "filter", "", "Specify which terminators to validate")
 	return validateTerminatorsCmd
 }
@@ -103,7 +103,7 @@ func (self *validateTerminatorsAction) validateTerminators(cmd *cobra.Command, _
 			fmt.Printf("channel closed, exiting")
 			return nil
 		case detail := <-self.eventNotify:
-			if !self.showOnlyInvalid || detail.State != mgmt_pb.TerminatorState_Valid {
+			if self.includeValid || detail.State != mgmt_pb.TerminatorState_Valid {
 				fmt.Printf("id: %s, binding: %s, hostId: %s, routerId: %s, state: %s, fixed: %v, detail: %s\n",
 					detail.TerminatorId, detail.Binding, detail.HostId, detail.RouterId, detail.State.String(), detail.Fixed, detail.Detail)
 			}
diff --git a/zititest/go.mod b/zititest/go.mod
index 6245b9427..377636e53 100644
--- a/zititest/go.mod
+++ b/zititest/go.mod
@@ -14,11 +14,11 @@ require (
 	github.com/michaelquigley/pfxlog v0.6.10
 	github.com/openziti/agent v1.0.16
 	github.com/openziti/channel/v2 v2.0.119
-	github.com/openziti/edge-api v0.26.11
+	github.com/openziti/edge-api v0.26.12
 	github.com/openziti/fablab v0.5.42
 	github.com/openziti/foundation/v2 v2.0.37
 	github.com/openziti/identity v1.0.70
-	github.com/openziti/sdk-golang v0.22.28
+	github.com/openziti/sdk-golang v0.23.3
 	github.com/openziti/storage v0.2.30
 	github.com/openziti/transport/v2 v2.0.122
 	github.com/openziti/ziti v0.28.3
@@ -27,7 +27,7 @@ require (
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/cobra v1.8.0
 	github.com/stretchr/testify v1.8.4
-	go.etcd.io/bbolt v1.3.8
+	go.etcd.io/bbolt v1.3.9
 	golang.org/x/net v0.21.0
 	google.golang.org/protobuf v1.32.0
 	gopkg.in/yaml.v2 v2.4.0
@@ -66,7 +66,7 @@ require (
 	github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa // indirect
 	github.com/gaissmai/extnetip v0.4.0 // indirect
 	github.com/go-acme/lego/v4 v4.15.0 // indirect
-	github.com/go-jose/go-jose/v3 v3.0.1 // indirect
+	github.com/go-jose/go-jose/v3 v3.0.3 // indirect
 	github.com/go-logr/logr v1.4.1 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.3.0 // indirect
@@ -76,7 +76,7 @@ require (
 	github.com/go-openapi/jsonreference v0.20.4 // indirect
 	github.com/go-openapi/loads v0.21.5 // indirect
 	github.com/go-openapi/spec v0.20.14 // indirect
-	github.com/go-openapi/strfmt v0.22.0 // indirect
+	github.com/go-openapi/strfmt v0.22.1 // indirect
 	github.com/go-openapi/swag v0.22.9 // indirect
 	github.com/go-openapi/validate v0.23.0 // indirect
 	github.com/go-resty/resty/v2 v2.11.0 // indirect
@@ -94,7 +94,7 @@ require (
 	github.com/hashicorp/golang-lru v0.6.0 // indirect
 	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
-	github.com/hashicorp/raft v1.6.0 // indirect
+	github.com/hashicorp/raft v1.6.1 // indirect
 	github.com/hashicorp/raft-boltdb v0.0.0-20220329195025-15018e9b97e0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/influxdata/influxdb-client-go/v2 v2.13.0 // indirect
@@ -179,20 +179,20 @@ require (
 	github.com/zitadel/oidc/v2 v2.12.0 // indirect
 	go.mongodb.org/mongo-driver v1.14.0 // indirect
 	go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 // indirect
-	go.opentelemetry.io/otel v1.23.1 // indirect
-	go.opentelemetry.io/otel/metric v1.23.1 // indirect
-	go.opentelemetry.io/otel/trace v1.23.1 // indirect
+	go.opentelemetry.io/otel v1.24.0 // indirect
+	go.opentelemetry.io/otel/metric v1.24.0 // indirect
+	go.opentelemetry.io/otel/trace v1.24.0 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.9.0 // indirect
 	go4.org v0.0.0-20180809161055-417644f6feb5 // indirect
-	golang.org/x/crypto v0.19.0 // indirect
+	golang.org/x/crypto v0.21.0 // indirect
 	golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
 	golang.org/x/image v0.13.0 // indirect
 	golang.org/x/mod v0.14.0 // indirect
 	golang.org/x/oauth2 v0.16.0 // indirect
 	golang.org/x/sync v0.6.0 // indirect
-	golang.org/x/sys v0.17.0 // indirect
-	golang.org/x/term v0.17.0 // indirect
+	golang.org/x/sys v0.18.0 // indirect
+	golang.org/x/term v0.18.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/tools v0.17.0 // indirect
 	google.golang.org/appengine v1.6.8 // indirect
diff --git a/zititest/go.sum b/zititest/go.sum
index dd53cdd95..3a3f66eb1 100644
--- a/zititest/go.sum
+++ b/zititest/go.sum
@@ -209,8 +209,8 @@ github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-jose/go-jose/v3 v3.0.1 h1:pWmKFVtt+Jl0vBZTIpz/eAKwsm6LkIxDVVbFHKkchhA=
-github.com/go-jose/go-jose/v3 v3.0.1/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8=
+github.com/go-jose/go-jose/v3 v3.0.3 h1:fFKWeig/irsp7XD2zBxvnmA/XaRWp5V3CBsZXJF7G7k=
+github.com/go-jose/go-jose/v3 v3.0.3/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
@@ -237,8 +237,8 @@ github.com/go-openapi/runtime v0.27.1 h1:ae53yaOoh+fx/X5Eaq8cRmavHgDma65XPZuvBqv
 github.com/go-openapi/runtime v0.27.1/go.mod h1:fijeJEiEclyS8BRurYE1DE5TLb9/KZl6eAdbzjsrlLU=
 github.com/go-openapi/spec v0.20.14 h1:7CBlRnw+mtjFGlPDRZmAMnq35cRzI91xj03HVyUi/Do=
 github.com/go-openapi/spec v0.20.14/go.mod h1:8EOhTpBoFiask8rrgwbLC3zmJfz4zsCUueRuPM6GNkw=
-github.com/go-openapi/strfmt v0.22.0 h1:Ew9PnEYc246TwrEspvBdDHS4BVKXy/AOVsfqGDgAcaI=
-github.com/go-openapi/strfmt v0.22.0/go.mod h1:HzJ9kokGIju3/K6ap8jL+OlGAbjpSv27135Yr9OivU4=
+github.com/go-openapi/strfmt v0.22.1 h1:5Ky8cybT4576C6Ffc+8gYji/wRXCo6Ozm8RaWjPI6jc=
+github.com/go-openapi/strfmt v0.22.1/go.mod h1:OfVoytIXJasDkkGvkb1Cceb3BPyMOwk1FgmyyEw7NYg=
 github.com/go-openapi/swag v0.22.9 h1:XX2DssF+mQKM2DHsbgZK74y/zj4mo9I99+89xUmuZCE=
 github.com/go-openapi/swag v0.22.9/go.mod h1:3/OXnFfnMAwBD099SwYRk7GD3xOrr1iL7d/XNLXVVwE=
 github.com/go-openapi/validate v0.23.0 h1:2l7PJLzCis4YUGEoW6eoQw3WhyM65WSIcjX6SQnlfDw=
@@ -389,8 +389,8 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO
 github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
 github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
 github.com/hashicorp/raft v1.1.0/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM=
-github.com/hashicorp/raft v1.6.0 h1:tkIAORZy2GbJ2Trp5eUSggLXDPOJLXC+JJLNMMqtgtM=
-github.com/hashicorp/raft v1.6.0/go.mod h1:Xil5pDgeGwRWuX4uPUmwa+7Vagg4N804dz6mhNi6S7o=
+github.com/hashicorp/raft v1.6.1 h1:v/jm5fcYHvVkL0akByAp+IDdDSzCNCGhdO6VdB56HIM=
+github.com/hashicorp/raft v1.6.1/go.mod h1:N1sKh6Vn47mrWvEArQgILTyng8GoDRNYlgKyK7PMjs0=
 github.com/hashicorp/raft-boltdb v0.0.0-20220329195025-15018e9b97e0 h1:CO8dBMLH6dvE1jTn/30ZZw3iuPsNfajshWoJTnVc5cc=
 github.com/hashicorp/raft-boltdb v0.0.0-20220329195025-15018e9b97e0/go.mod h1:nTakvJ4XYq45UXtn0DbwR4aU9ZdjlnIenpbs6Cd+FM0=
 github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
@@ -590,8 +590,8 @@ github.com/openziti/channel/v2 v2.0.119 h1:stfSrnDqoTi78LMvQA3+NSivHjQnRrYKrgij5
 github.com/openziti/channel/v2 v2.0.119/go.mod h1:lSRJwqmbkE34DgXYEmUhVCzwcQcx65vZGE8nuBNK458=
 github.com/openziti/dilithium v0.3.3 h1:PLgQ6PMNLSTzCFbX/h98cmudgz/cU6TmjdSv5NAPD8k=
 github.com/openziti/dilithium v0.3.3/go.mod h1:vsCjI2AU/hon9e+dLhUFbCNGesJDj2ASgkySOcpmvjo=
-github.com/openziti/edge-api v0.26.11 h1:qINsfGpPBTnbuDrOq+qcMZuBdlXosqvHX7sQhLA+cM4=
-github.com/openziti/edge-api v0.26.11/go.mod h1:30SiUmR+9gOBi9HUZgXLpCO2nNCbNFVx2jwXV2Dh4Og=
+github.com/openziti/edge-api v0.26.12 h1:5VRz0cWtfQq2rhSA7Ne6amM7YNI6pQGRfNgbKt0g6kQ=
+github.com/openziti/edge-api v0.26.12/go.mod h1:tKZRUFDB9zM5J1zBS0ok2r40OhJqWykZaU9HSBQgr8w=
 github.com/openziti/fablab v0.5.42 h1:vENJKfEba2T4sSLwlKDL/IzBYfY8iHnhc4umf6IESiY=
 github.com/openziti/fablab v0.5.42/go.mod h1:HDT06y1QX8kO8ZQrgHvZmJsvc8iRybESGtlDLDII4ks=
 github.com/openziti/foundation/v2 v2.0.37 h1:7pa4vWrlwllEoLXaK2rx91AffLQJ8k5pvc92oWANavA=
@@ -604,8 +604,8 @@ github.com/openziti/metrics v1.2.45 h1:+3zqszLWyFdTgzbsQD90V0yJcC9Ek77qKaIGMQXkA
 github.com/openziti/metrics v1.2.45/go.mod h1:g6CgAEbFes2UtdfGrsR8AKkuoBVL5dkU61843uQvllM=
 github.com/openziti/runzmd v1.0.38 h1:0eFWAf7R9Thx99ue7Gql2dBavsqv4jldU8W5AIZJ8wo=
 github.com/openziti/runzmd v1.0.38/go.mod h1:WiQi+YIXxZBH1bwjr+eo/e3ftfTLEABpN0i2QIhBI9w=
-github.com/openziti/sdk-golang v0.22.28 h1:s159CT42dXug4GiJiN/kM6/ol+N2LFZ2tUk6bOpbgiI=
-github.com/openziti/sdk-golang v0.22.28/go.mod h1:BLaLvcLqAgf3JFoDPWLTj3j3X5rndo6ZejdDdkMlihQ=
+github.com/openziti/sdk-golang v0.23.3 h1:LujHMOcGoY7qeT4mtcJhzklEG2f192islXjT/jY0oQ4=
+github.com/openziti/sdk-golang v0.23.3/go.mod h1:2yXIT3b6iWVyCChwuavaJHhlHozrxoM2IB3bybbVX4M=
 github.com/openziti/secretstream v0.1.16 h1:tVanF7OpJL1MJ1gvWaRlR2i+kAbrGsxr3q6EXFOS08U=
 github.com/openziti/secretstream v0.1.16/go.mod h1:bvjGBUW/0e5MzD5S3FW3rhGASRNWAi+kTkTENZ9qRDE=
 github.com/openziti/storage v0.2.30 h1:15o8rSSgtcNCSBONt81ZRASQXJHh2L9Y0aWcHoQ81dE=
@@ -824,8 +824,8 @@ github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQ
 github.com/zitadel/oidc/v2 v2.12.0 h1:4aMTAy99/4pqNwrawEyJqhRb3yY3PtcDxnoDSryhpn4=
 github.com/zitadel/oidc/v2 v2.12.0/go.mod h1:LrRav74IiThHGapQgCHZOUNtnqJG0tcZKHro/91rtLw=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
-go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA=
-go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
+go.etcd.io/bbolt v1.3.9 h1:8x7aARPEXiXbHmtUwAIv7eV2fQFHrLLavdiJ3uzJXoI=
+go.etcd.io/bbolt v1.3.9/go.mod h1:zaO32+Ti0PK1ivdPtgMESzuzL2VPoIG1PCQNvOdo/dE=
 go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
 go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
 go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ=
@@ -841,14 +841,14 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
 go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
-go.opentelemetry.io/otel v1.23.1 h1:Za4UzOqJYS+MUczKI320AtqZHZb7EqxO00jAHE0jmQY=
-go.opentelemetry.io/otel v1.23.1/go.mod h1:Td0134eafDLcTS4y+zQ26GE8u3dEuRBiBCTUIRHaikA=
-go.opentelemetry.io/otel/metric v1.23.1 h1:PQJmqJ9u2QaJLBOELl1cxIdPcpbwzbkjfEyelTl2rlo=
-go.opentelemetry.io/otel/metric v1.23.1/go.mod h1:mpG2QPlAfnK8yNhNJAxDZruU9Y1/HubbC+KyH8FaCWI=
+go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo=
+go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo=
+go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI=
+go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco=
 go.opentelemetry.io/otel/sdk v1.17.0 h1:FLN2X66Ke/k5Sg3V623Q7h7nt3cHXaW1FOvKKrW0IpE=
 go.opentelemetry.io/otel/sdk v1.17.0/go.mod h1:U87sE0f5vQB7hwUoW98pW5Rz4ZDuCFBZFNUBlSgmDFQ=
-go.opentelemetry.io/otel/trace v1.23.1 h1:4LrmmEd8AU2rFvU1zegmvqW7+kWarxtNOPyeL6HmYY8=
-go.opentelemetry.io/otel/trace v1.23.1/go.mod h1:4IpnpJFwr1mo/6HL8XIPJaE9y0+u1KcVmuW7dwFSVrI=
+go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI=
+go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
@@ -876,7 +876,6 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@@ -885,8 +884,9 @@ golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
-golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
 golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
+golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
+golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1099,16 +1099,18 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
 golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
-golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U=
 golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
+golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8=
+golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
diff --git a/zititest/models/db-sdk-hosting-test/configs/ctrl.yml.tmpl b/zititest/models/db-sdk-hosting-test/configs/ctrl.yml.tmpl
new file mode 100644
index 000000000..850bbc302
--- /dev/null
+++ b/zititest/models/db-sdk-hosting-test/configs/ctrl.yml.tmpl
@@ -0,0 +1,205 @@
+v: 3
+
+{{if .Component.GetFlag "ha"}}
+raft:
+  minClusterSize: 3
+  dataDir: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/ctrldata
+{{else}}
+db: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/ctrl.db
+{{end}}
+
+identity:
+  cert: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.Id }}/certs/{{ .Component.Id }}-server.cert
+  key: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.Id }}/keys/{{ .Component.Id }}-server.key
+  ca: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.Id }}/certs/{{ .Component.Id }}-server.chain.pem
+
+commandRateLimiter:
+    enabled:   true
+    maxQueued: 25
+
+# the endpoint that routers will connect to the controller over.
+ctrl:
+  listener: tls:0.0.0.0:6262
+  options:
+    advertiseAddress: tls:{{ .Host.PublicIp }}:6262
+    # (optional) settings
+    # set the maximum number of connect requests that are buffered and waiting to be acknowledged (1 to 5000, default 1000)
+    #maxQueuedConnects:      50
+
+    # the maximum number of connects that have  begun hello synchronization (1 to 1000, default 16)
+    #maxOutstandingConnects: 100
+
+    # the number of milliseconds to wait before a hello synchronization fails and closes the connection (30ms to 60000ms, default: 1000ms)
+    #connectTimeoutMs:       3000
+
+    # Sets the control channel write timeout. A write timeout will close the control channel, so the router will reconnect
+    #writeTimeout: 15s
+
+    # A listener address which will be sent to connecting routers in order to change their configured controller
+    # address. If defined, routers will update address configuration to immediately use the new address for future
+    # connections. The value of newListener must be resolvable both via DNS and validate via certificates
+    #newListener: tls:localhost:6262
+
+events:
+  jsonLogger:
+    subscriptions:
+      - type: entityChange
+      - type: edge.apiSessions
+      - type: edge.entityCounts
+        interval: 15s
+      - type: edge.sessions
+      - type: fabric.routers
+      - type: fabric.terminators
+#      - type: metrics
+#        sourceFilter: .*
+#        metricFilter: .*egress.*m1_rate*
+#      - type: fabric.circuits
+#        include:
+#          - created
+#        include:
+#          - created
+#      - type: fabric.usage
+#      - type: services
+#      - type: fabric.usage
+    handler:
+      type: file
+      format: json
+      path: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/logs/event.log
+
+healthChecks:
+  boltCheck:
+    # How often to try entering a bolt read tx. Defaults to 30 seconds
+    interval: 30s
+    # When to timeout the check. Defaults to 15 seconds
+    timeout: 15s
+    # How long to wait before starting the check. Defaults to 15 seconds
+    initialDelay: 15s
+
+# By having an 'edge' section defined, the ziti-controller will attempt to parse the edge configuration. Removing this
+# section, commenting out, or altering the name of the section will cause the edge to not run.
+edge:
+  # This section represents the configuration of the Edge API that is served over HTTPS
+  api:
+    #(optional, default 90s) Alters how frequently heartbeat and last activity values are persisted
+    # activityUpdateInterval: 90s
+    #(optional, default 250) The number of API Sessions updated for last activity per transaction
+    # activityUpdateBatchSize: 250
+    # sessionTimeout - optional, default 10m
+    # The number of minutes before an Edge API session will timeout. Timeouts are reset by
+    # API requests and connections that are maintained to Edge Routers
+    sessionTimeout: 30m
+    # address - required
+    # The default address (host:port) to use for enrollment for the Client API. This value must match one of the addresses
+    # defined in a bind point's address field for the `edge-client` API in the web section.
+    address: {{ .Host.PublicIp }}:1280
+  # enrollment - required
+  # A section containing settings pertaining to enrollment.
+  enrollment:
+    # signingCert - required
+    # A Ziti Identity configuration section that specifically makes use of the cert and key fields to define
+    # a signing certificate from the PKI that the Ziti environment is using to sign certificates. The signingCert.cert
+    # will be added to the /.well-known CA store that is used to bootstrap trust with the Ziti Controller.
+    signingCert:
+      cert: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.Id }}/certs/{{ .Component.Id }}.cert
+      key: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.Id }}/keys/{{ .Component.Id }}.key
+
+    # edgeIdentity - optional
+    # A section for identity enrollment specific settings
+    edgeIdentity:
+      # duration - optional, default 5m
+      # The length of time that a Ziti Edge Identity enrollment should remain valid. After
+      # this duration, the enrollment will expire and not longer be usable.
+      duration: 1h
+    # edgeRouter - Optional
+    # A section for edge router enrollment specific settings.
+    edgeRouter:
+      # duration - optional, default 5m
+      # The length of time that a Ziti Edge Router enrollment should remain valid. After
+      # this duration, the enrollment will expire and not longer be usable.
+      duration: 1h
+
+
+# web - optional
+# Defines webListeners that will be hosted by the controller. Each webListener can host many APIs and be bound to many
+# bind points.
+web:
+  # name - required
+  # Provides a name for this listener, used for logging output. Not required to be unique, but is highly suggested.
+  - name: all-apis-localhost
+    # bindPoints - required
+    # One or more bind points are required. A bind point specifies an interface (interface:port string) that defines
+    # where on the host machine the webListener will listen and the address (host:port) that should be used to
+    # publicly address the webListener(i.e. mydomain.com, localhost, 127.0.0.1). This public address may be used for
+    # incoming address resolution as well as used in responses in the API.
+    bindPoints:
+      #interface - required
+      # A host:port string on which network interface to listen on. 0.0.0.0 will listen on all interfaces
+      - interface: 0.0.0.0:1280
+
+        # address - required
+        # The public address that external incoming requests will be able to resolve. Used in request processing and
+        # response content that requires full host:port/path addresses.
+        address: {{ .Host.PublicIp }}:1280
+
+        # newAddress - optional
+        # A host:port string which will be sent out as an HTTP header "ziti-new-address" if specified. If the header
+        # is present, clients should update location configuration to immediately use the new address for future
+        # connections. The value of newAddress must be resolvable both via DNS and validate via certificates
+        #newAddress: localhost:1280
+    # identity - optional
+    # Allows the webListener to have a specific identity instead of defaulting to the root `identity` section.
+    #    identity:
+    #      cert:                 ${ZITI_SOURCE}/ziti/etc/ca/intermediate/certs/ctrl-client.cert.pem
+    #      server_cert:          ${ZITI_SOURCE}/ziti/etc/ca/intermediate/certs/ctrl-server.cert.pem
+    #      key:                  ${ZITI_SOURCE}/ziti/etc/ca/intermediate/private/ctrl.key.pem
+    #      ca:                   ${ZITI_SOURCE}/ziti/etc/ca/intermediate/certs/ca-chain.cert.pem
+    # options - optional
+    # Allows the specification of webListener level options - mainly dealing with HTTP/TLS settings. These options are
+    # used for all http servers started by the current webListener.
+    options:
+      # idleTimeout - optional, default 5000ms
+      # The maximum amount of idle time in milliseconds allowed for pipelined HTTP requests. Setting this too high
+      # can cause resources on the host to be consumed as clients remain connected and idle. Lowering this value
+      # will cause clients to reconnect on subsequent HTTPs requests.
+      idleTimeout: 5000ms  #http timeouts, new
+
+      # readTimeout - optional, default 5000ms
+      # The maximum amount of time in milliseconds http servers will wait to read the first incoming requests. A higher
+      # value risks consuming resources on the host with clients that are acting bad faith or suffering from high latency
+      # or packet loss. A lower value can risk losing connections to high latency/packet loss clients.
+
+      readTimeout: 5000ms
+      # writeTimeout - optional, default 10000ms
+      # The total maximum time in milliseconds that the http server will wait for a single requests to be received and
+      # responded too. A higher value can allow long running requests to consume resources on the host. A lower value
+      # can risk ending requests before the server has a chance to respond.
+
+      writeTimeout: 100000ms
+      # minTLSVersion - optional, default TSL1.2
+      # The minimum version of TSL to support
+
+      minTLSVersion: TLS1.2
+      # maxTLSVersion - optional, default TSL1.3
+      # The maximum version of TSL to support
+
+      maxTLSVersion: TLS1.3
+    # apis - required
+    # Allows one or more APIs to be bound to this webListener
+    apis:
+      # binding - required
+      # Specifies an API to bind to this webListener. Built-in APIs are
+      #   - health-checks
+      #   - edge-management
+      #   - edge-client
+      #   - fabric-management
+      - binding: health-checks
+        options: {}
+      - binding: fabric
+      - binding: edge-management
+        # options - variable optional/required
+        # This section is used to define values that are specified by the API they are associated with.
+        # These settings are per API. The example below is for the `edge-api` and contains both optional values and
+        # required values.
+        options: {}
+      - binding: edge-client
+        options: {}
diff --git a/zititest/models/db-sdk-hosting-test/configs/router.yml.tmpl b/zititest/models/db-sdk-hosting-test/configs/router.yml.tmpl
new file mode 100644
index 000000000..72999dd24
--- /dev/null
+++ b/zititest/models/db-sdk-hosting-test/configs/router.yml.tmpl
@@ -0,0 +1,70 @@
+{{$ssh_username := .Model.MustVariable "credentials.ssh.username"}}
+{{$identity := .Component.Id}}
+{{$router_ip := .Host.PublicIp}}
+  
+v: 3
+
+enableDebugOps: true
+
+identity:
+  cert:                 /home/{{$ssh_username}}/fablab/cfg/{{$identity}}-client.cert
+  server_cert:          /home/{{$ssh_username}}/fablab/cfg/{{$identity}}-server.cert
+  key:                  /home/{{$ssh_username}}/fablab/cfg/{{$identity}}.key
+  ca:                   /home/{{$ssh_username}}/fablab/cfg/{{$identity}}-server.chain.pem
+
+ctrl:
+  endpoints: {{ range $host := .Model.MustSelectHosts "component.ctrl" 1 }}
+    - tls:{{ $host.PublicIp }}:6262{{end}}
+
+healthChecks:
+  ctrlPingCheck:
+    # How often to ping the controller over the control channel. Defaults to 30 seconds
+    interval: 30s
+    # When to timeout the ping. Defaults to 15 seconds
+    timeout: 15s
+    # How long to wait before pinging the controller. Defaults to 15 seconds
+    initialDelay: 15s
+
+metrics:
+  reportInterval: 15s
+  messageQueueSize: 10
+
+link:
+  listeners:
+    - binding:          transport
+      bind:             tls:0.0.0.0:6000
+      advertise:        tls:{{$router_ip}}:6000
+  dialers:
+    - binding:          transport
+
+listeners:
+{{if .Component.HasTag "tunneler"}}
+  - binding: tunnel
+    options:
+      mode: host
+{{end}}
+  - binding: edge
+    address: tls:0.0.0.0:6262
+    options:
+      # (required) The public hostname and port combination that Ziti SDKs should connect on. Previously this was in the chanIngress section.
+      advertise: {{ .Host.PublicIp }}:6262
+
+# By having an 'edge' section defined, the ziti-router will attempt to parse the edge configuration. Removing this
+# section, commenting out, or altering the name of the section will cause the router to no longer operate as an Edge
+# Router.
+edge:
+  # (required) Information used to generate the initial registration CSR. For documentation on these fields please
+  # refer to the openssl documentation. These values MUST be supplied and have no defaults.
+  csr:
+    country: US
+    province: NC
+    locality: Charlotte
+    organization: NetFoundry
+    organizationalUnit: Ziti
+
+    # (required) SANs that this Gateways certs should contain. At least one IP or DNS SAN should be defined that matches
+    # the edge listeners "advertise" value from the "listeners" section.
+    sans:
+      ip:
+        - {{ .Host.PublicIp }}
+
diff --git a/zititest/models/db-sdk-hosting-test/main.go b/zititest/models/db-sdk-hosting-test/main.go
new file mode 100644
index 000000000..9ff26d6a3
--- /dev/null
+++ b/zititest/models/db-sdk-hosting-test/main.go
@@ -0,0 +1,324 @@
+package main
+
+import (
+	"embed"
+	_ "embed"
+	"fmt"
+	"github.com/openziti/fablab"
+	"github.com/openziti/fablab/kernel/lib/actions"
+	"github.com/openziti/fablab/kernel/lib/actions/component"
+	"github.com/openziti/fablab/kernel/lib/actions/host"
+	"github.com/openziti/fablab/kernel/lib/actions/semaphore"
+	"github.com/openziti/fablab/kernel/lib/binding"
+	"github.com/openziti/fablab/kernel/lib/runlevel/0_infrastructure/aws_ssh_key"
+	"github.com/openziti/fablab/kernel/lib/runlevel/0_infrastructure/semaphore"
+	"github.com/openziti/fablab/kernel/lib/runlevel/0_infrastructure/terraform"
+	distribution "github.com/openziti/fablab/kernel/lib/runlevel/3_distribution"
+	"github.com/openziti/fablab/kernel/lib/runlevel/3_distribution/rsync"
+	aws_ssh_key2 "github.com/openziti/fablab/kernel/lib/runlevel/6_disposal/aws_ssh_key"
+	"github.com/openziti/fablab/kernel/lib/runlevel/6_disposal/terraform"
+	"github.com/openziti/fablab/kernel/model"
+	"github.com/openziti/fablab/resources"
+	"github.com/openziti/ziti/controller/db"
+	"github.com/openziti/ziti/zititest/models/test_resources"
+	"github.com/openziti/ziti/zititest/zitilab"
+	"github.com/openziti/ziti/zititest/zitilab/actions/edge"
+	"github.com/openziti/ziti/zititest/zitilab/models"
+	"go.etcd.io/bbolt"
+	"os"
+	"path"
+	"strings"
+	"time"
+)
+
+// const TargetZitiVersion = "v0.31.0"
+
+const TargetZitiVersion = ""
+const TargetZitiEdgeTunnelVersion = ""
+
+//const TargetZitiEdgeTunnelVersion = "0.22.12"
+
+var TunnelType = "!zet"
+
+//go:embed configs
+var configResource embed.FS
+
+type dbStrategy struct{}
+
+func (d dbStrategy) GetDbFile(m *model.Model) string {
+	return m.MustStringVariable("db_file")
+}
+
+func (d dbStrategy) GetSite(router *db.EdgeRouter) (string, bool) {
+	if strings.Contains(strings.ToLower(router.Name), "london") {
+		return "eu-west-2a", true // london region
+	}
+	if strings.Contains(strings.ToLower(router.Name), "virginia") {
+		return "us-east-1a", true // london region
+	}
+	if strings.Contains(strings.ToLower(router.Name), "melbourne") {
+		return "ap-southeast-2a", true // sydney region
+	}
+
+	return "us-east-1a", true
+}
+
+func (d dbStrategy) PostProcess(router *db.EdgeRouter, c *model.Component) {
+	if router.IsTunnelerEnabled {
+		c.Scope.Tags = append(c.Scope.Tags, "tunneler")
+	}
+	c.Scope.Tags = append(c.Scope.Tags, "edge-router")
+	c.Scope.Tags = append(c.Scope.Tags, "pre-created")
+	c.Host.InstanceType = "c5.xlarge"
+	c.Type.(*zitilab.RouterType).Version = TargetZitiVersion
+}
+
+func (d dbStrategy) ProcessDbModel(tx *bbolt.Tx, m *model.Model, builder *models.ZitiDbBuilder) error {
+	if err := builder.CreateEdgeRouterHosts(tx, m); err != nil {
+		return err
+	}
+	return d.CreateIdentityHosts(tx, m, builder)
+}
+
+func (d dbStrategy) CreateIdentityHosts(tx *bbolt.Tx, m *model.Model, builder *models.ZitiDbBuilder) error {
+	stores := builder.GetStores()
+	ids, _, err := stores.Identity.QueryIds(tx, "true limit none")
+	if err != nil {
+		return err
+	}
+
+	servicesCount := 0
+	hostingIdentities := map[string]int{}
+
+	for _, identityId := range ids {
+		cursorProvider := stores.Identity.GetIdentityServicesCursorProvider(identityId)
+		cursor := cursorProvider(tx, true)
+		identityServiceCount := 0
+		for cursor.IsValid() {
+			serviceId := string(cursor.Current())
+			if stores.EdgeService.IsBindableByIdentity(tx, serviceId, identityId) {
+				identityServiceCount++
+			}
+			cursor.Next()
+		}
+		if identityServiceCount > 0 {
+			servicesCount += identityServiceCount
+			hostingIdentities[identityId] = identityServiceCount
+		}
+	}
+
+	fmt.Printf("service count: %v\n", servicesCount)
+
+	regionCount := len(m.Regions)
+
+	perRegion := servicesCount / regionCount
+	idIdx := 0
+
+	avgTunnelsPerHost := 15
+
+	m.RangeSortedRegions(func(regionId string, region *model.Region) {
+		regionServiceCount := 0
+
+		var regionIdentityIds []string
+
+		for {
+			if idIdx >= len(ids) {
+				break
+			}
+			identityId := ids[idIdx]
+			idIdx++
+
+			svcCount, found := hostingIdentities[identityId]
+			if !found {
+				continue
+			}
+			regionServiceCount += svcCount
+			regionIdentityIds = append(regionIdentityIds, identityId)
+			if regionServiceCount > perRegion {
+				break
+			}
+		}
+
+		hostCount := len(regionIdentityIds) / avgTunnelsPerHost
+		var hosts []*model.Host
+
+		for i := 0; i < hostCount; i++ {
+			tunnelsHost := &model.Host{
+				Scope:        model.Scope{Tags: model.Tags{}},
+				Region:       region,
+				Components:   model.Components{},
+				InstanceType: "t3.xlarge",
+			}
+			hostId := fmt.Sprintf("%s_svc_hosts_%v", regionId, i)
+			region.Hosts[hostId] = tunnelsHost
+			hosts = append(hosts, tunnelsHost)
+		}
+
+		hostIdx := 0
+		for _, identityId := range regionIdentityIds {
+			tunnelHost := hosts[hostIdx%len(hosts)]
+			hostIdx++
+
+			svcCount := hostingIdentities[identityId]
+
+			getConfigPath := func(c *model.Component) string {
+				user := c.GetHost().GetSshUser()
+				return fmt.Sprintf("/home/%s/etc/%s.json", user, c.Id)
+			}
+
+			var tunnelType model.ComponentType
+			if TunnelType == "zet" {
+				tunnelType = &zitilab.ZitiEdgeTunnelType{
+					Version:     TargetZitiEdgeTunnelVersion,
+					LogConfig:   "'2;bind.c=6'",
+					ConfigPathF: getConfigPath,
+				}
+			} else {
+				tunnelType = &zitilab.ZitiTunnelType{
+					Mode:        zitilab.ZitiTunnelModeHost,
+					Version:     TargetZitiVersion,
+					ConfigPathF: getConfigPath,
+				}
+			}
+
+			tunnelComponent := &model.Component{
+				Scope: model.Scope{Tags: model.Tags{"client", "pre-created", fmt.Sprintf("serviceCount=%v", svcCount)}},
+				Type:  tunnelType,
+				Host:  tunnelHost,
+			}
+			tunnelHost.Components[identityId] = tunnelComponent
+		}
+	})
+
+	return nil
+}
+
+var dbStrategyInstance = dbStrategy{}
+
+var m = &model.Model{
+	Id: "db-sdk-hosting-test",
+	Scope: model.Scope{
+		Defaults: model.Variables{
+			"environment": "db-sdk-hosting-test",
+			"credentials": model.Variables{
+				"aws": model.Variables{
+					"managed_key": true,
+				},
+				"ssh": model.Variables{
+					"username": "ubuntu",
+				},
+				"edge": model.Variables{
+					"username": "admin",
+					"password": "admin",
+				},
+			},
+			"metrics": model.Variables{
+				"influxdb": model.Variables{
+					"url": "http://localhost:8086",
+					"db":  "ziti",
+				},
+			},
+		},
+	},
+	StructureFactories: []model.Factory{
+		&models.ZitiDbBuilder{Strategy: dbStrategyInstance},
+	},
+	Resources: model.Resources{
+		resources.Configs:   resources.SubFolder(configResource, "configs"),
+		resources.Binaries:  os.DirFS(path.Join(os.Getenv("GOPATH"), "bin")),
+		resources.Terraform: test_resources.TerraformResources(),
+	},
+	Regions: model.Regions{
+		"us-east-1": {
+			Region: "us-east-1",
+			Site:   "us-east-1a",
+			Hosts: model.Hosts{
+				"ctrl": {
+					InstanceType: "c5.xlarge",
+					Components: model.Components{
+						"ctrl": {
+							Scope: model.Scope{Tags: model.Tags{"ctrl"}},
+							Type: &zitilab.ControllerType{
+								Version: TargetZitiVersion,
+							},
+						},
+					},
+				},
+			},
+		},
+	},
+
+	Actions: model.ActionBinders{
+		"bootstrap": model.ActionBinder(func(m *model.Model) model.Action {
+			workflow := actions.Workflow()
+
+			workflow.AddAction(component.Start("#ctrl"))
+			workflow.AddAction(semaphore.Sleep(2 * time.Second))
+
+			workflow.AddAction(edge.Login("#ctrl"))
+
+			workflow.AddAction(edge.ReEnrollEdgeRouters(".edge-router .pre-created", 2))
+			if quickRun, _ := m.GetBoolVariable("quick_run"); !quickRun {
+				workflow.AddAction(edge.ReEnrollIdentities(".client .pre-created", 10))
+			}
+			return workflow
+		}),
+		"stop": model.Bind(component.StopInParallelHostExclusive("*", 15)),
+		"clean": model.Bind(actions.Workflow(
+			component.StopInParallelHostExclusive("*", 15),
+			host.GroupExec("*", 25, "rm -f logs/*"),
+		)),
+		"login": model.Bind(edge.Login("#ctrl")),
+		"restart": model.ActionBinder(func(run *model.Model) model.Action {
+			workflow := actions.Workflow()
+			workflow.AddAction(component.StopInParallel("*", 100))
+			workflow.AddAction(component.Start(".ctrl"))
+			workflow.AddAction(semaphore.Sleep(2 * time.Second))
+			workflow.AddAction(component.StartInParallel(".edge-router", 10))
+			workflow.AddAction(semaphore.Sleep(2 * time.Second))
+			workflow.AddAction(component.StartInParallel(".client", 50))
+			return workflow
+		}),
+	},
+
+	Infrastructure: model.Stages{
+		aws_ssh_key.Express(),
+		&terraform_0.Terraform{
+			Retries: 3,
+			ReadyCheck: &semaphore_0.ReadyStage{
+				MaxWait: 90 * time.Second,
+			},
+		},
+	},
+
+	Distribution: model.Stages{
+		distribution.DistributeSshKey("*"),
+		rsync.RsyncStaged(),
+		model.StageActionF(func(run model.Run) error {
+			quickRun, _ := run.GetModel().GetBoolVariable("quick_run")
+			_, targetedSync := run.GetModel().Scope.GetVariable("sync.target")
+
+			if !quickRun && !targetedSync {
+				dbFile := dbStrategyInstance.GetDbFile(run.GetModel())
+				deferred := rsync.NewRsyncHost("#ctrl", dbFile, "/home/ubuntu/ctrl.db")
+				return deferred.Execute(run)
+			}
+			return nil
+		}),
+	},
+
+	Disposal: model.Stages{
+		terraform.Dispose(),
+		aws_ssh_key2.Dispose(),
+	},
+}
+
+func main() {
+	m.AddActivationActions("stop", "bootstrap")
+
+	model.AddBootstrapExtension(binding.AwsCredentialsLoader)
+	model.AddBootstrapExtension(aws_ssh_key.KeyManager)
+
+	fablab.InitModel(m)
+	fablab.Run()
+}
diff --git a/zititest/models/links-test/main.go b/zititest/models/links-test/main.go
index 86d892b3e..09d9207e0 100644
--- a/zititest/models/links-test/main.go
+++ b/zititest/models/links-test/main.go
@@ -116,7 +116,6 @@ var m = &model.Model{
 			Site:   "us-east-1a",
 			Hosts: model.Hosts{
 				"ctrl1": {
-					InstanceType: "t3.medium",
 					Components: model.Components{
 						"ctrl1": {
 							Scope: model.Scope{Tags: model.Tags{"ctrl"}},
diff --git a/zititest/models/sdk-hosting-test/configs/ctrl.yml.tmpl b/zititest/models/sdk-hosting-test/configs/ctrl.yml.tmpl
index 5a134a115..850bbc302 100644
--- a/zititest/models/sdk-hosting-test/configs/ctrl.yml.tmpl
+++ b/zititest/models/sdk-hosting-test/configs/ctrl.yml.tmpl
@@ -13,6 +13,10 @@ identity:
   key: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.Id }}/keys/{{ .Component.Id }}-server.key
   ca: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.Id }}/certs/{{ .Component.Id }}-server.chain.pem
 
+commandRateLimiter:
+    enabled:   true
+    maxQueued: 25
+
 # the endpoint that routers will connect to the controller over.
 ctrl:
   listener: tls:0.0.0.0:6262
diff --git a/zititest/models/sdk-hosting-test/main.go b/zititest/models/sdk-hosting-test/main.go
index 175660e99..b1d173b01 100644
--- a/zititest/models/sdk-hosting-test/main.go
+++ b/zititest/models/sdk-hosting-test/main.go
@@ -3,14 +3,15 @@ package main
 import (
 	"embed"
 	_ "embed"
-	"errors"
 	"fmt"
+	"github.com/michaelquigley/pfxlog"
 	"github.com/openziti/fablab"
 	"github.com/openziti/fablab/kernel/lib/actions"
 	"github.com/openziti/fablab/kernel/lib/actions/component"
 	"github.com/openziti/fablab/kernel/lib/actions/host"
 	"github.com/openziti/fablab/kernel/lib/actions/semaphore"
 	"github.com/openziti/fablab/kernel/lib/binding"
+	"github.com/openziti/fablab/kernel/lib/parallel"
 	"github.com/openziti/fablab/kernel/lib/runlevel/0_infrastructure/aws_ssh_key"
 	"github.com/openziti/fablab/kernel/lib/runlevel/0_infrastructure/semaphore"
 	"github.com/openziti/fablab/kernel/lib/runlevel/0_infrastructure/terraform"
@@ -20,16 +21,15 @@ import (
 	"github.com/openziti/fablab/kernel/lib/runlevel/6_disposal/terraform"
 	"github.com/openziti/fablab/kernel/model"
 	"github.com/openziti/fablab/resources"
-	"github.com/openziti/ziti/controller/db"
 	"github.com/openziti/ziti/zititest/models/test_resources"
 	"github.com/openziti/ziti/zititest/zitilab"
+	zitilib_actions "github.com/openziti/ziti/zititest/zitilab/actions"
 	"github.com/openziti/ziti/zititest/zitilab/actions/edge"
+	"github.com/openziti/ziti/zititest/zitilab/chaos"
+	"github.com/openziti/ziti/zititest/zitilab/cli"
 	"github.com/openziti/ziti/zititest/zitilab/models"
-	"go.etcd.io/bbolt"
 	"os"
-	"os/exec"
 	"path"
-	"strings"
 	"time"
 )
 
@@ -45,158 +45,34 @@ var TunnelType = "!zet"
 //go:embed configs
 var configResource embed.FS
 
-type dbStrategy struct{}
+type scaleStrategy struct{}
 
-func (d dbStrategy) GetDbFile(m *model.Model) string {
-	return m.MustStringVariable("db_file")
-}
-
-func (d dbStrategy) GetSite(router *db.EdgeRouter) (string, bool) {
-	if strings.Contains(strings.ToLower(router.Name), "london") {
-		return "eu-west-2a", true // london region
-	}
-	if strings.Contains(strings.ToLower(router.Name), "virginia") {
-		return "us-east-1a", true // london region
-	}
-	if strings.Contains(strings.ToLower(router.Name), "melbourne") {
-		return "ap-southeast-2a", true // sydney region
-	}
-
-	return "us-east-1a", true
-}
-
-func (d dbStrategy) PostProcess(router *db.EdgeRouter, c *model.Component) {
-	if router.IsTunnelerEnabled {
-		c.Scope.Tags = append(c.Scope.Tags, "tunneler")
-	}
-	c.Scope.Tags = append(c.Scope.Tags, "edge-router")
-	c.Scope.Tags = append(c.Scope.Tags, "pre-created")
-	c.Host.InstanceType = "c5.xlarge"
-	c.Type.(*zitilab.RouterType).Version = TargetZitiVersion
-}
-
-func (d dbStrategy) ProcessDbModel(tx *bbolt.Tx, m *model.Model, builder *models.ZitiDbBuilder) error {
-	if err := builder.CreateEdgeRouterHosts(tx, m); err != nil {
-		return err
+func (self scaleStrategy) IsScaled(entity model.Entity) bool {
+	if entity.GetType() == model.EntityTypeHost {
+		return entity.GetScope().HasTag("router") || entity.GetScope().HasTag("host")
 	}
-	return d.CreateIdentityHosts(tx, m, builder)
+	return entity.GetType() == model.EntityTypeComponent && entity.GetScope().HasTag("host")
 }
 
-func (d dbStrategy) CreateIdentityHosts(tx *bbolt.Tx, m *model.Model, builder *models.ZitiDbBuilder) error {
-	stores := builder.GetStores()
-	ids, _, err := stores.Identity.QueryIds(tx, "true limit none")
-	if err != nil {
-		return err
-	}
-
-	servicesCount := 0
-	hostingIdentities := map[string]int{}
-
-	for _, identityId := range ids {
-		cursorProvider := stores.Identity.GetIdentityServicesCursorProvider(identityId)
-		cursor := cursorProvider(tx, true)
-		identityServiceCount := 0
-		for cursor.IsValid() {
-			serviceId := string(cursor.Current())
-			if stores.EdgeService.IsBindableByIdentity(tx, serviceId, identityId) {
-				identityServiceCount++
-			}
-			cursor.Next()
+func (self scaleStrategy) GetEntityCount(entity model.Entity) uint32 {
+	if entity.GetType() == model.EntityTypeHost {
+		if entity.GetScope().HasTag("router") {
+			return 2
 		}
-		if identityServiceCount > 0 {
-			servicesCount += identityServiceCount
-			hostingIdentities[identityId] = identityServiceCount
-		}
-	}
-
-	fmt.Printf("service count: %v\n", servicesCount)
-
-	regionCount := len(m.Regions)
-
-	perRegion := servicesCount / regionCount
-	idIdx := 0
-
-	avgTunnelsPerHost := 15
-
-	m.RangeSortedRegions(func(regionId string, region *model.Region) {
-		regionServiceCount := 0
-
-		var regionIdentityIds []string
-
-		for {
-			if idIdx >= len(ids) {
-				break
-			}
-			identityId := ids[idIdx]
-			idIdx++
-
-			svcCount, found := hostingIdentities[identityId]
-			if !found {
-				continue
-			}
-			regionServiceCount += svcCount
-			regionIdentityIds = append(regionIdentityIds, identityId)
-			if regionServiceCount > perRegion {
-				break
+		if entity.GetScope().HasTag("host") {
+			h := entity.(*model.Host)
+			if h.Region.Id == "us-east-1" {
+				return 8
 			}
+			return 6
 		}
-
-		hostCount := len(regionIdentityIds) / avgTunnelsPerHost
-		var hosts []*model.Host
-
-		for i := 0; i < hostCount; i++ {
-			tunnelsHost := &model.Host{
-				Scope:        model.Scope{Tags: model.Tags{}},
-				Region:       region,
-				Components:   model.Components{},
-				InstanceType: "t3.xlarge",
-			}
-			hostId := fmt.Sprintf("%s_svc_hosts_%v", regionId, i)
-			region.Hosts[hostId] = tunnelsHost
-			hosts = append(hosts, tunnelsHost)
-		}
-
-		hostIdx := 0
-		for _, identityId := range regionIdentityIds {
-			tunnelHost := hosts[hostIdx%len(hosts)]
-			hostIdx++
-
-			svcCount := hostingIdentities[identityId]
-
-			getConfigPath := func(c *model.Component) string {
-				user := c.GetHost().GetSshUser()
-				return fmt.Sprintf("/home/%s/etc/%s.json", user, c.Id)
-			}
-
-			var tunnelType model.ComponentType
-			if TunnelType == "zet" {
-				tunnelType = &zitilab.ZitiEdgeTunnelType{
-					Version:     TargetZitiEdgeTunnelVersion,
-					LogConfig:   "'2;bind.c=6'",
-					ConfigPathF: getConfigPath,
-				}
-			} else {
-				tunnelType = &zitilab.ZitiTunnelType{
-					Mode:        zitilab.ZitiTunnelModeHost,
-					Version:     TargetZitiVersion,
-					ConfigPathF: getConfigPath,
-				}
-			}
-
-			tunnelComponent := &model.Component{
-				Scope: model.Scope{Tags: model.Tags{"sdk-tunneler", "pre-created", fmt.Sprintf("serviceCount=%v", svcCount)}},
-				Type:  tunnelType,
-				Host:  tunnelHost,
-			}
-			tunnelHost.Components[identityId] = tunnelComponent
-		}
-	})
-
-	return nil
+	}
+	if entity.GetType() == model.EntityTypeComponent {
+		return 10
+	}
+	return 1
 }
 
-var dbStrategyInstance = dbStrategy{}
-
 var m = &model.Model{
 	Id: "sdk-hosting-test",
 	Scope: model.Scope{
@@ -223,7 +99,31 @@ var m = &model.Model{
 		},
 	},
 	StructureFactories: []model.Factory{
-		&models.ZitiDbBuilder{Strategy: dbStrategyInstance},
+		model.FactoryFunc(func(m *model.Model) error {
+			err := m.ForEachHost("component.router", 1, func(host *model.Host) error {
+				host.InstanceType = "c5.xlarge"
+				return nil
+			})
+
+			if err != nil {
+				return err
+			}
+
+			err = m.ForEachComponent(".host", 1, func(c *model.Component) error {
+				c.Type.(*zitilab.ZitiTunnelType).Mode = zitilab.ZitiTunnelModeHost
+				return nil
+			})
+
+			if err != nil {
+				return err
+			}
+
+			return m.ForEachHost("component.host", 1, func(host *model.Host) error {
+				host.InstanceType = "c5.xlarge"
+				return nil
+			})
+		}),
+		model.NewScaleFactoryWithDefaultEntityFactory(&scaleStrategy{}),
 	},
 	Resources: model.Resources{
 		resources.Configs:   resources.SubFolder(configResource, "configs"),
@@ -235,10 +135,10 @@ var m = &model.Model{
 			Region: "us-east-1",
 			Site:   "us-east-1a",
 			Hosts: model.Hosts{
-				"ctrl": {
+				"ctrl1": {
 					InstanceType: "c5.xlarge",
 					Components: model.Components{
-						"ctrl": {
+						"ctrl1": {
 							Scope: model.Scope{Tags: model.Tags{"ctrl"}},
 							Type: &zitilab.ControllerType{
 								Version: TargetZitiVersion,
@@ -246,6 +146,84 @@ var m = &model.Model{
 						},
 					},
 				},
+				"router-us-{{.ScaleIndex}}": {
+					Scope: model.Scope{Tags: model.Tags{"router"}},
+					Components: model.Components{
+						"router-us-{{.Host.ScaleIndex}}": {
+							Scope: model.Scope{Tags: model.Tags{"router"}},
+							Type: &zitilab.RouterType{
+								Version: TargetZitiVersion,
+							},
+						},
+					},
+				},
+				"host-us-{{ .ScaleIndex }}": {
+					Scope: model.Scope{Tags: model.Tags{"host"}},
+					Components: model.Components{
+						"host-us-{{ .Host.ScaleIndex }}-{{ .ScaleIndex }}": {
+							Scope: model.Scope{Tags: model.Tags{"host"}},
+							Type: &zitilab.ZitiTunnelType{
+								Version: TargetZitiVersion,
+							},
+						},
+					},
+				},
+			},
+		},
+		"eu-west-2": {
+			Region: "us-west-2",
+			Site:   "us-west-2a",
+			Hosts: model.Hosts{
+				"router-eu-{{.ScaleIndex}}": {
+					Scope: model.Scope{Tags: model.Tags{"router"}},
+					Components: model.Components{
+						"router-eu-{{.Host.ScaleIndex}}": {
+							Scope: model.Scope{Tags: model.Tags{"router"}},
+							Type: &zitilab.RouterType{
+								Version: TargetZitiVersion,
+							},
+						},
+					},
+				},
+				"host-eu-{{ .ScaleIndex }}": {
+					Scope: model.Scope{Tags: model.Tags{"host"}},
+					Components: model.Components{
+						"host-eu-{{ .Host.ScaleIndex }}-{{ .ScaleIndex }}": {
+							Scope: model.Scope{Tags: model.Tags{"host"}},
+							Type: &zitilab.ZitiTunnelType{
+								Version: TargetZitiVersion,
+							},
+						},
+					},
+				},
+			},
+		},
+		"ap-southeast-2": {
+			Region: "ap-southeast-2",
+			Site:   "ap-southeast-2a",
+			Hosts: model.Hosts{
+				"router-ap-{{.ScaleIndex}}": {
+					Scope: model.Scope{Tags: model.Tags{"router", "scaled"}},
+					Components: model.Components{
+						"router-ap-{{.Host.ScaleIndex}}": {
+							Scope: model.Scope{Tags: model.Tags{"router"}},
+							Type: &zitilab.RouterType{
+								Version: TargetZitiVersion,
+							},
+						},
+					},
+				},
+				"host-ap-{{ .ScaleIndex }}": {
+					Scope: model.Scope{Tags: model.Tags{"host", "scaled"}},
+					Components: model.Components{
+						"host-ap-{{ .Host.ScaleIndex }}-{{ .ScaleIndex }}": {
+							Scope: model.Scope{Tags: model.Tags{"host"}},
+							Type: &zitilab.ZitiTunnelType{
+								Version: TargetZitiVersion,
+							},
+						},
+					},
+				},
 			},
 		},
 	},
@@ -254,15 +232,80 @@ var m = &model.Model{
 		"bootstrap": model.ActionBinder(func(m *model.Model) model.Action {
 			workflow := actions.Workflow()
 
-			workflow.AddAction(component.Start("#ctrl"))
-			workflow.AddAction(semaphore.Sleep(2 * time.Second))
+			isHA := len(m.SelectComponents(".ctrl")) > 1
+
+			workflow.AddAction(component.StopInParallel("*", 300))
+			workflow.AddAction(host.GroupExec("*", 25, "rm -f logs/* ctrl.db"))
+			workflow.AddAction(host.GroupExec("component.ctrl", 5, "rm -rf ./fablab/ctrldata"))
+
+			if !isHA {
+				workflow.AddAction(component.Exec("#ctrl1", zitilab.ControllerActionInitStandalone))
+			}
 
-			workflow.AddAction(edge.Login("#ctrl"))
+			workflow.AddAction(component.Start(".ctrl"))
 
-			workflow.AddAction(edge.ReEnrollEdgeRouters(".edge-router .pre-created", 2))
-			if quickRun, _ := m.GetBoolVariable("quick_run"); !quickRun {
-				workflow.AddAction(edge.ReEnrollIdentities(".sdk-tunneler .pre-created", 10))
+			if isHA {
+				workflow.AddAction(semaphore.Sleep(2 * time.Second))
+				workflow.AddAction(edge.RaftJoin(".ctrl"))
+				workflow.AddAction(semaphore.Sleep(2 * time.Second))
+				workflow.AddAction(edge.InitRaftController("#ctrl1"))
 			}
+
+			workflow.AddAction(edge.ControllerAvailable("#ctrl1", 30*time.Second))
+
+			workflow.AddAction(edge.Login("#ctrl1"))
+
+			workflow.AddAction(edge.InitEdgeRouters(models.RouterTag, 25))
+			workflow.AddAction(edge.InitIdentities(".host", 25))
+
+			workflow.AddAction(zitilib_actions.Edge("create", "edge-router-policy", "all", "--edge-router-roles", "#all", "--identity-roles", "#all"))
+			workflow.AddAction(zitilib_actions.Edge("create", "service-edge-router-policy", "all", "--service-roles", "#all", "--edge-router-roles", "#all"))
+
+			workflow.AddAction(zitilib_actions.Edge("create", "config", "host-config", "host.v1", `
+				{
+					"address" : "localhost",
+					"port" : 8080,
+					"protocol" : "tcp"
+				}`))
+
+			workflow.AddAction(model.ActionFunc(func(run model.Run) error {
+				var tasks []parallel.Task
+				for i := 0; i < 2000; i++ {
+					name := fmt.Sprintf("service-%04d", i)
+					task := func() error {
+						_, err := cli.Exec(run.GetModel(), "edge", "create", "service", name, "-c", "host-config")
+						return err
+					}
+					tasks = append(tasks, task)
+				}
+				return parallel.Execute(tasks, 25)
+			}))
+
+			workflow.AddAction(model.ActionFunc(func(run model.Run) error {
+				identities := getHostNames()
+				serviceIdx := 0
+				var tasks []parallel.Task
+				for i, identity := range identities {
+					name := fmt.Sprintf("service-policy-%03d", i)
+					identityRoles := fmt.Sprintf("@%s", identity)
+					servicesRoles := ""
+					for j := 0; j < 10; j++ {
+						idx := serviceIdx % 2000
+						if j > 0 {
+							servicesRoles += ","
+						}
+						servicesRoles += fmt.Sprintf("@service-%04d", idx)
+						serviceIdx++
+					}
+					tasks = append(tasks, func() error {
+						_, err := cli.Exec(run.GetModel(), "edge", "create", "service-policy", name, "Bind",
+							"--identity-roles", identityRoles, "--service-roles", servicesRoles)
+						return err
+					})
+				}
+				return parallel.Execute(tasks, 25)
+			}))
+
 			return workflow
 		}),
 		"stop": model.Bind(component.StopInParallelHostExclusive("*", 15)),
@@ -270,45 +313,36 @@ var m = &model.Model{
 			component.StopInParallelHostExclusive("*", 15),
 			host.GroupExec("*", 25, "rm -f logs/*"),
 		)),
-		"login": model.Bind(edge.Login("#ctrl")),
-		"refreshCtrlZiti": model.ActionBinder(func(m *model.Model) model.Action {
-			return model.ActionFunc(func(run model.Run) error {
-				zitiPath, err := exec.LookPath("ziti")
-				if err != nil {
-					return err
-				}
-
-				deferred := rsync.NewRsyncHost("ctrl", zitiPath, "/home/ubuntu/fablab/bin/ziti")
-				return deferred.Execute(run)
-			})
-		}),
-		"refreshRouterZiti": model.ActionBinder(func(m *model.Model) model.Action {
-			return model.ActionFunc(func(run model.Run) error {
-				zitiPath, err := exec.LookPath("ziti")
-				if err != nil {
-					return err
-				}
-
-				deferred := rsync.NewRsyncHost("component.edge-router", zitiPath, "/home/ubuntu/fablab/bin/ziti")
-				return deferred.Execute(run)
-			})
+		"login": model.Bind(edge.Login("#ctrl1")),
+		"restart": model.ActionBinder(func(run *model.Model) model.Action {
+			workflow := actions.Workflow()
+			workflow.AddAction(component.StopInParallel("*", 100))
+			workflow.AddAction(host.GroupExec("*", 25, "rm -f logs/*"))
+			workflow.AddAction(component.Start(".ctrl"))
+			workflow.AddAction(semaphore.Sleep(2 * time.Second))
+			workflow.AddAction(component.StartInParallel(".router", 10))
+			workflow.AddAction(semaphore.Sleep(2 * time.Second))
+			workflow.AddAction(component.StartInParallel(".host", 50))
+			return workflow
 		}),
-		"refreshZiti": model.ActionBinder(func(m *model.Model) model.Action {
-			return model.ActionFunc(func(run model.Run) error {
-				zitiPath, err := exec.LookPath("ziti")
-				if err != nil {
-					return err
-				}
-
-				hosts := os.Getenv("HOSTS")
-				if hosts == "" {
-					return errors.New("expected hosts to refresh in HOSTS env")
-				}
-
-				deferred := rsync.NewRsyncHost(hosts, zitiPath, "/home/ubuntu/fablab/bin/ziti")
-				return deferred.Execute(run)
+		"sowChaos": model.Bind(model.ActionFunc(sowChaos)),
+		"validateUp": model.Bind(model.ActionFunc(func(run model.Run) error {
+			if err := chaos.ValidateUp(run, ".ctrl", 3, 15*time.Second); err != nil {
+				return err
+			}
+			err := run.GetModel().ForEachComponent(".ctrl", 3, func(c *model.Component) error {
+				return edge.ControllerAvailable(c.Id, 30*time.Second).Execute(run)
 			})
-		}),
+			if err != nil {
+				return err
+			}
+			if err := chaos.ValidateUp(run, ".router", 100, time.Minute); err != nil {
+				pfxlog.Logger().WithError(err).Error("validate up failed, trying to start all routers again")
+				return component.StartInParallel(".router", 100).Execute(run)
+			}
+			return nil
+		})),
+		"validate": model.Bind(model.ActionFunc(validateTerminators)),
 	},
 
 	Infrastructure: model.Stages{
@@ -324,14 +358,6 @@ var m = &model.Model{
 	Distribution: model.Stages{
 		distribution.DistributeSshKey("*"),
 		rsync.RsyncStaged(),
-		model.StageActionF(func(run model.Run) error {
-			if quickRun, _ := run.GetModel().GetBoolVariable("quick_run"); !quickRun {
-				dbFile := dbStrategyInstance.GetDbFile(run.GetModel())
-				deferred := rsync.NewRsyncHost("#ctrl", dbFile, "/home/ubuntu/ctrl.db")
-				return deferred.Execute(run)
-			}
-			return nil
-		}),
 	},
 
 	Disposal: model.Stages{
@@ -340,6 +366,20 @@ var m = &model.Model{
 	},
 }
 
+func getHostNames() []string {
+	var result []string
+	for i := 0; i < 8; i++ {
+		for j := 0; j < 10; j++ {
+			result = append(result, fmt.Sprintf("host-us-%d-%d", i, j))
+			if i < 6 {
+				result = append(result, fmt.Sprintf("host-eu-%d-%d", i, j))
+				result = append(result, fmt.Sprintf("host-ap-%d-%d", i, j))
+			}
+		}
+	}
+	return result
+}
+
 func main() {
 	m.AddActivationActions("stop", "bootstrap")
 
diff --git a/zititest/models/sdk-hosting-test/validation.go b/zititest/models/sdk-hosting-test/validation.go
new file mode 100644
index 000000000..f1fcc4ac6
--- /dev/null
+++ b/zititest/models/sdk-hosting-test/validation.go
@@ -0,0 +1,263 @@
+/*
+	Copyright NetFoundry Inc.
+
+	Licensed under the Apache License, Version 2.0 (the "License");
+	you may not use this file except in compliance with the License.
+	You may obtain a copy of the License at
+
+	https://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+	distributed under the License is distributed on an "AS IS" BASIS,
+	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+	See the License for the specific language governing permissions and
+	limitations under the License.
+*/
+
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"github.com/michaelquigley/pfxlog"
+	"github.com/openziti/channel/v2"
+	"github.com/openziti/channel/v2/protobufs"
+	"github.com/openziti/fablab/kernel/model"
+	"github.com/openziti/ziti/common/pb/mgmt_pb"
+	"github.com/openziti/ziti/controller/rest_client/terminator"
+	"github.com/openziti/ziti/zititest/zitilab/chaos"
+	"github.com/openziti/ziti/zititest/zitilab/zitirest"
+	"google.golang.org/protobuf/proto"
+	"math/rand"
+	"time"
+)
+
+// start with a random scenario then cycle through them
+var scenarioCounter = rand.Intn(7)
+
+func sowChaos(run model.Run) error {
+	var controllers []*model.Component
+	var err error
+
+	scenarioCounter = (scenarioCounter + 1) % 7
+	scenario := scenarioCounter + 1
+
+	if scenario&0b001 > 0 {
+		controllers, err = chaos.SelectRandom(run, ".ctrl", chaos.RandomOfTotal())
+		if err != nil {
+			return err
+		}
+		time.Sleep(5 * time.Second)
+	}
+
+	var routers []*model.Component
+	if scenario&0b010 > 0 {
+		routers, err = chaos.SelectRandom(run, ".router", chaos.PercentageRange(10, 75))
+		if err != nil {
+			return err
+		}
+	}
+
+	var hosts []*model.Component
+	if scenario&0b100 > 0 {
+		hosts, err = chaos.SelectRandom(run, ".host", chaos.PercentageRange(10, 75))
+		if err != nil {
+			return err
+		}
+	}
+
+	toRestart := append(([]*model.Component)(nil), controllers...)
+	toRestart = append(toRestart, routers...)
+	toRestart = append(toRestart, hosts...)
+	fmt.Printf("restarting %d controllers,  %d routers and %d hosts\n", len(controllers), len(routers), len(hosts))
+	return chaos.RestartSelected(run, toRestart, 100)
+}
+
+func validateTerminators(run model.Run) error {
+	ctrls := run.GetModel().SelectComponents(".ctrl")
+	errC := make(chan error, len(ctrls))
+	deadline := time.Now().Add(15 * time.Minute)
+	for _, ctrl := range ctrls {
+		ctrlComponent := ctrl
+		go validateTerminatorsForCtrlWithChan(ctrlComponent, deadline, errC)
+	}
+
+	for i := 0; i < len(ctrls); i++ {
+		err := <-errC
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func validateTerminatorsForCtrlWithChan(c *model.Component, deadline time.Time, errC chan<- error) {
+	errC <- validateTerminatorsForCtrl(c, deadline)
+}
+
+func validateTerminatorsForCtrl(c *model.Component, deadline time.Time) error {
+	expectedTerminatorCount := int64(6000)
+	username := c.MustStringVariable("credentials.edge.username")
+	password := c.MustStringVariable("credentials.edge.password")
+	edgeApiBaseUrl := c.Host.PublicIp + ":1280"
+
+	var clients *zitirest.Clients
+	loginStart := time.Now()
+	for {
+		var err error
+		clients, err = zitirest.NewManagementClients(edgeApiBaseUrl)
+		if err != nil {
+			if time.Since(loginStart) > time.Minute {
+				return err
+			}
+			pfxlog.Logger().WithField("ctrlId", c.Id).WithError(err).Info("failed to initialize mgmt client, trying again in 1s")
+			time.Sleep(time.Second)
+			continue
+		}
+
+		if err = clients.Authenticate(username, password); err != nil {
+			if time.Since(loginStart) > time.Minute {
+				return err
+			}
+			pfxlog.Logger().WithField("ctrlId", c.Id).WithError(err).Info("failed to authenticate, trying again in 1s")
+			time.Sleep(time.Second)
+		} else {
+			break
+		}
+	}
+
+	terminatorsPresent := false
+	start := time.Now()
+
+	logger := pfxlog.Logger().WithField("ctrl", c.Id)
+	var lastLog time.Time
+	for time.Now().Before(deadline) && !terminatorsPresent {
+		terminatorCount, err := getTerminatorCount(clients)
+		if err != nil {
+			return nil
+		}
+		if terminatorCount == expectedTerminatorCount {
+			terminatorsPresent = true
+		} else {
+			time.Sleep(5 * time.Second)
+		}
+		if time.Since(lastLog) > time.Minute {
+			logger.Infof("current terminator count: %v, elapsed time: %v", terminatorCount, time.Since(start))
+			lastLog = time.Now()
+		}
+	}
+
+	if terminatorsPresent {
+		logger.Infof("all terminators present, elapsed time: %v", time.Since(start))
+	} else {
+		return fmt.Errorf("fail to reach expected terminator count of %v on controller %v", expectedTerminatorCount, c.Id)
+	}
+
+	for {
+		count, err := validateRouterSdkTerminators(c.Id, clients)
+		if err == nil {
+			return nil
+		}
+
+		if time.Now().After(deadline) {
+			return err
+		}
+
+		logger.Infof("current count of invalid sdk terminators: %v, elapsed time: %v", count, time.Since(start))
+		time.Sleep(15 * time.Second)
+	}
+}
+
+func getTerminatorCount(clients *zitirest.Clients) (int64, error) {
+	ctx, cancelF := context.WithTimeout(context.Background(), 15*time.Second)
+	defer cancelF()
+
+	filter := "limit 1"
+	result, err := clients.Fabric.Terminator.ListTerminators(&terminator.ListTerminatorsParams{
+		Filter:  &filter,
+		Context: ctx,
+	})
+
+	if err != nil {
+		return 0, err
+	}
+	count := *result.Payload.Meta.Pagination.TotalCount
+	return count, nil
+}
+
+func validateRouterSdkTerminators(id string, clients *zitirest.Clients) (int, error) {
+	logger := pfxlog.Logger().WithField("ctrl", id)
+
+	closeNotify := make(chan struct{})
+	eventNotify := make(chan *mgmt_pb.RouterSdkTerminatorsDetails, 1)
+
+	handleSdkTerminatorResults := func(msg *channel.Message, _ channel.Channel) {
+		detail := &mgmt_pb.RouterSdkTerminatorsDetails{}
+		if err := proto.Unmarshal(msg.Body, detail); err != nil {
+			pfxlog.Logger().WithError(err).Error("unable to unmarshal router sdk terminator details")
+			return
+		}
+		eventNotify <- detail
+	}
+
+	bindHandler := func(binding channel.Binding) error {
+		binding.AddReceiveHandlerF(int32(mgmt_pb.ContentType_ValidateRouterSdkTerminatorsResultType), handleSdkTerminatorResults)
+		binding.AddCloseHandler(channel.CloseHandlerF(func(ch channel.Channel) {
+			close(closeNotify)
+		}))
+		return nil
+	}
+
+	ch, err := clients.NewWsMgmtChannel(channel.BindHandlerF(bindHandler))
+	if err != nil {
+		return 0, err
+	}
+
+	defer func() {
+		_ = ch.Close()
+	}()
+
+	request := &mgmt_pb.ValidateRouterSdkTerminatorsRequest{
+		Filter: "limit none",
+	}
+	responseMsg, err := protobufs.MarshalTyped(request).WithTimeout(10 * time.Second).SendForReply(ch)
+
+	response := &mgmt_pb.ValidateRouterSdkTerminatorsResponse{}
+	if err = protobufs.TypedResponse(response).Unmarshall(responseMsg, err); err != nil {
+		return 0, err
+	}
+
+	if !response.Success {
+		return 0, fmt.Errorf("failed to start sdk terminator validation: %s", response.Message)
+	}
+
+	logger.Infof("started validation of %v routers", response.RouterCount)
+
+	expected := response.RouterCount
+
+	invalid := 0
+	for expected > 0 {
+		select {
+		case <-closeNotify:
+			fmt.Printf("channel closed, exiting")
+			return 0, errors.New("unexpected close of mgmt channel")
+		case routerDetail := <-eventNotify:
+			if !routerDetail.ValidateSuccess {
+				return invalid, fmt.Errorf("error: unable to validate on controller %s (%s)", routerDetail.Message, id)
+			}
+			for _, linkDetail := range routerDetail.Details {
+				if !linkDetail.IsValid {
+					invalid++
+				}
+			}
+			expected--
+		}
+	}
+	if invalid == 0 {
+		logger.Infof("sdk terminator validation of %v routers successful", response.RouterCount)
+		return invalid, nil
+	}
+	return invalid, fmt.Errorf("invalid sdk terminators found")
+}
diff --git a/zititest/zitilab/actions/edge/init_identities.go b/zititest/zitilab/actions/edge/init_identities.go
index 3eae38a20..47faa42bd 100644
--- a/zititest/zitilab/actions/edge/init_identities.go
+++ b/zititest/zitilab/actions/edge/init_identities.go
@@ -31,7 +31,7 @@ func (action *initIdentitiesAction) createAndEnrollIdentity(run model.Run, c *mo
 
 	jwtFileName := filepath.Join(run.GetTmpDir(), c.Id+".jwt")
 
-	err := zitilib_actions.EdgeExec(c.GetModel(), "create", "identity", "service", c.Id,
+	err := zitilib_actions.EdgeExec(c.GetModel(), "create", "identity", c.Id,
 		"--jwt-output-file", jwtFileName,
 		"-a", strings.Join(c.Tags, ","))
 
diff --git a/zititest/zitilab/cli/cli.go b/zititest/zitilab/cli/cli.go
index 6b463884d..dfd4b1e4f 100644
--- a/zititest/zitilab/cli/cli.go
+++ b/zititest/zitilab/cli/cli.go
@@ -44,3 +44,44 @@ func Exec(m *model.Model, args ...string) (string, error) {
 
 	return cliOut.String(), nil
 }
+
+func NewSeq() *Seq {
+	return &Seq{}
+}
+
+type Seq struct {
+	err error
+}
+
+func (self *Seq) Error() error {
+	return self.err
+}
+
+func (self *Seq) Args(args ...string) []string {
+	return args
+}
+
+func (self *Seq) Exec(args ...string) {
+	self.ExecF(args, nil)
+}
+
+func (self *Seq) ExecF(args []string, f func(string) error) {
+	if self.err != nil {
+		return
+	}
+
+	var cliOut bytes.Buffer
+	var cliErr bytes.Buffer
+
+	ziticli := cmd.NewRootCommand(os.Stdin, &cliOut, &cliErr)
+	ziticli.SetArgs(args)
+	logrus.Infof("executing: %s", strings.Join(args, " "))
+	if err := ziticli.Execute(); err != nil {
+		logrus.Errorf("err executing command, err:[%e]", err)
+		self.err = err
+	}
+
+	if self.err == nil && f != nil {
+		self.err = f(cliOut.String())
+	}
+}
diff --git a/zititest/zitilab/component_common.go b/zititest/zitilab/component_common.go
index 6ac064322..31d23922a 100644
--- a/zititest/zitilab/component_common.go
+++ b/zititest/zitilab/component_common.go
@@ -27,10 +27,11 @@ import (
 
 func getZitiProcessFilter(c *model.Component, zitiType string) func(string) bool {
 	return func(s string) bool {
-		return strings.Contains(s, "ziti") &&
+		matches := strings.Contains(s, "ziti") &&
 			strings.Contains(s, zitiType) &&
 			strings.Contains(s, fmt.Sprintf("--cli-agent-alias %s ", c.Id)) &&
 			!strings.Contains(s, "sudo ")
+		return matches
 	}
 }
 
diff --git a/zititest/zitilab/component_ziti_tunnel.go b/zititest/zitilab/component_ziti_tunnel.go
index daca6af97..9ddab77ac 100644
--- a/zititest/zitilab/component_ziti_tunnel.go
+++ b/zititest/zitilab/component_ziti_tunnel.go
@@ -80,11 +80,18 @@ func (self *ZitiTunnelType) StageFiles(r model.Run, c *model.Component) error {
 
 func (self *ZitiTunnelType) InitializeHost(_ model.Run, c *model.Component) error {
 	if self.Mode == ZitiTunnelModeTproxy {
-		cmds := []string{
-			"sudo sed -i 's/#DNS=/DNS=127.0.0.1/g' /etc/systemd/resolved.conf",
-			"sudo systemctl restart systemd-resolved",
+		key := "ziti_tunnel.resolve_setup_done"
+		if _, found := c.Host.Data[key]; !found {
+			cmds := []string{
+				"sudo sed -i 's/#DNS=/DNS=127.0.0.1/g' /etc/systemd/resolved.conf",
+				"sudo systemctl restart systemd-resolved",
+			}
+			if err := c.Host.ExecLogOnlyOnError(cmds...); err != nil {
+				return err
+			}
+			c.Host.Data[key] = true
+			return nil
 		}
-		return c.Host.ExecLogOnlyOnError(cmds...)
 	}
 	return nil
 }
@@ -122,8 +129,8 @@ func (self *ZitiTunnelType) Start(_ model.Run, c *model.Component) error {
 		useSudo = "sudo"
 	}
 
-	serviceCmd := fmt.Sprintf("%s %s tunnel %s -v --log-formatter pfxlog -i %s --cli-agent-alias %s > %s 2>&1 &",
-		useSudo, binaryPath, mode.String(), configPath, c.Id, logsPath)
+	serviceCmd := fmt.Sprintf("%s %s tunnel %s -v --cli-agent-alias %s --log-formatter pfxlog -i %s > %s 2>&1 &",
+		useSudo, binaryPath, mode.String(), c.Id, configPath, logsPath)
 
 	value, err := c.Host.ExecLogged(
 		"rm -f "+logsPath,
diff --git a/zititest/zitilab/component_zrok_looptest.go b/zititest/zitilab/component_zrok_looptest.go
index 6b529ee5e..26c6b971e 100644
--- a/zititest/zitilab/component_zrok_looptest.go
+++ b/zititest/zitilab/component_zrok_looptest.go
@@ -86,7 +86,7 @@ func (self *ZrokLoopTestType) Start(_ model.Run, c *model.Component) error {
 	logsPath := fmt.Sprintf("/home/%s/logs/%s.log", user, c.Id)
 
 	maxDwell := 1000 + c.ScaleIndex
-	serviceCmd := fmt.Sprintf("nohup sudo -u %s %s test loop public --iterations %v --loopers %v --min-pacing-ms %v --max-pacing-ms %v "+
+	serviceCmd := fmt.Sprintf("nohup sudo -u %s %s test loop public -v --iterations %v --loopers %v --min-pacing-ms %v --max-pacing-ms %v "+
 		"--max-dwell-ms %d 2>&1 &> %s &",
 		userId, binaryPath, self.Iterations, self.Loopers, self.Pacing.Milliseconds(), self.Pacing.Milliseconds(), maxDwell, logsPath)
 
diff --git a/zititest/zitilab/models/db_builder.go b/zititest/zitilab/models/db_builder.go
index e05c15aff..f6938e6e9 100644
--- a/zititest/zitilab/models/db_builder.go
+++ b/zititest/zitilab/models/db_builder.go
@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"github.com/openziti/fablab/kernel/model"
 	"github.com/openziti/storage/boltz"
+	"github.com/openziti/ziti/controller/command"
 	"github.com/openziti/ziti/controller/db"
 	"github.com/openziti/ziti/controller/network"
 	"github.com/openziti/ziti/zititest/zitilab"
@@ -52,7 +53,7 @@ func (self *ZitiDbBuilder) Build(m *model.Model) error {
 		}
 	}()
 
-	self.stores, err = db.InitStores(self.zitiDb)
+	self.stores, err = db.InitStores(self.zitiDb, command.NoOpRateLimiter{})
 	if err != nil {
 		return errors.Wrapf(err, "unable to init fabric stores using db [%v]", dbFile)
 	}