diff --git a/.github/workflows/build-and-push-wasm-plugin-image.yaml b/.github/workflows/build-and-push-wasm-plugin-image.yaml index d716d5e39b..518dfeca75 100644 --- a/.github/workflows/build-and-push-wasm-plugin-image.yaml +++ b/.github/workflows/build-and-push-wasm-plugin-image.yaml @@ -89,7 +89,9 @@ jobs: push_command=${push_command%\"} # 删除PUSH_COMMAND中的双引号，确保oras push正常解析 target_image="${{ env.IMAGE_REGISTRY_SERVICE }}/${{ env.IMAGE_REPOSITORY}}/${{ env.PLUGIN_NAME }}:${{ env.VERSION }}" + target_image_latest="${{ env.IMAGE_REGISTRY_SERVICE }}/${{ env.IMAGE_REPOSITORY}}/${{ env.PLUGIN_NAME }}:latest" echo "TargetImage=${target_image}" + echo "TargetImageLatest=${target_image_latest}" cd ${{ github.workspace }}/plugins/wasm-go/extensions/${PLUGIN_NAME} if [ -f ./.buildrc ]; then @@ -108,7 +110,6 @@ jobs: tar czvf plugin.tar.gz plugin.wasm echo ${{ secrets.REGISTRY_PASSWORD }} | oras login -u ${{ secrets.REGISTRY_USERNAME }} --password-stdin ${{ env.IMAGE_REGISTRY_SERVICE }} oras push ${target_image} ${push_command} + oras push ${target_image_latest} ${push_command} " docker exec builder bash -c "$command" - - diff --git a/Makefile.core.mk b/Makefile.core.mk index 1f99882b5c..95341d0f5f 100644 --- a/Makefile.core.mk +++ b/Makefile.core.mk @@ -187,8 +187,8 @@ install: pre-install cd helm/higress; helm dependency build helm install higress helm/higress -n higress-system --create-namespace --set 'global.local=true' -ENVOY_LATEST_IMAGE_TAG ?= 7722dc383cd5d566d1b10a738f79a4cba1a18304 -ISTIO_LATEST_IMAGE_TAG ?= 7722dc383cd5d566d1b10a738f79a4cba1a18304 +ENVOY_LATEST_IMAGE_TAG ?= 2.0.1 +ISTIO_LATEST_IMAGE_TAG ?= 2.0.1 install-dev: pre-install helm install higress helm/core -n higress-system --create-namespace --set 'controller.tag=$(TAG)' --set 'gateway.replicas=1' --set 'pilot.tag=$(ISTIO_LATEST_IMAGE_TAG)' --set 'gateway.tag=$(ENVOY_LATEST_IMAGE_TAG)' --set 'global.local=true' diff --git a/VERSION b/VERSION index 46b105a30d..0ac852dded 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v2.0.0 +v2.0.1 diff --git a/api/kubernetes/customresourcedefinitions.gen.yaml b/api/kubernetes/customresourcedefinitions.gen.yaml index 1398b462f4..de7e1b9345 100644 --- a/api/kubernetes/customresourcedefinitions.gen.yaml +++ b/api/kubernetes/customresourcedefinitions.gen.yaml @@ -284,6 +284,10 @@ spec: type: string port: type: integer + protocol: + type: string + sni: + type: string type: type: string zkServicesPath: diff --git a/api/networking/v1/mcp_bridge.pb.go b/api/networking/v1/mcp_bridge.pb.go index 7aa8fbf28b..d71ea55025 100644 --- a/api/networking/v1/mcp_bridge.pb.go +++ b/api/networking/v1/mcp_bridge.pb.go @@ -126,6 +126,8 @@ type RegistryConfig struct { ConsulServiceTag string `protobuf:"bytes,15,opt,name=consulServiceTag,proto3" json:"consulServiceTag,omitempty"` ConsulRefreshInterval int64 `protobuf:"varint,16,opt,name=consulRefreshInterval,proto3" json:"consulRefreshInterval,omitempty"` AuthSecretName string `protobuf:"bytes,17,opt,name=authSecretName,proto3" json:"authSecretName,omitempty"` + Protocol string `protobuf:"bytes,18,opt,name=protocol,proto3" json:"protocol,omitempty"` + Sni string `protobuf:"bytes,19,opt,name=sni,proto3" json:"sni,omitempty"` } func (x *RegistryConfig) Reset() { @@ -279,6 +281,20 @@ func (x *RegistryConfig) GetAuthSecretName() string { return "" } +func (x *RegistryConfig) GetProtocol() string { + if x != nil { + return x.Protocol + } + return "" +} + +func (x *RegistryConfig) GetSni() string { + if x != nil { + return x.Sni + } + return "" +} + var File_networking_v1_mcp_bridge_proto protoreflect.FileDescriptor var file_networking_v1_mcp_bridge_proto_rawDesc = []byte{ @@ -292,7 +308,7 @@ var file_networking_v1_mcp_bridge_proto_rawDesc = []byte{ 0x69, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x68, 0x69, 0x67, 0x72, 0x65, 0x73, 0x73, 0x2e, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x52, 0x0a, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x69, 0x65, 0x73, 0x22, 0xa5, 0x05, 0x0a, + 0x52, 0x0a, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x69, 0x65, 0x73, 0x22, 0xd3, 0x05, 0x0a, 0x0e, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x17, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x03, 0xe0, 0x41, 0x02, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, @@ -335,10 +351,13 @@ var file_networking_v1_mcp_bridge_proto_rawDesc = []byte{ 0x72, 0x65, 0x73, 0x68, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x12, 0x26, 0x0a, 0x0e, 0x61, 0x75, 0x74, 0x68, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x11, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x61, 0x75, 0x74, 0x68, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, - 0x4e, 0x61, 0x6d, 0x65, 0x42, 0x2e, 0x5a, 0x2c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, - 0x6f, 0x6d, 0x2f, 0x61, 0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x2f, 0x68, 0x69, 0x67, 0x72, 0x65, - 0x73, 0x73, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x69, 0x6e, - 0x67, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, + 0x18, 0x12, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, + 0x12, 0x10, 0x0a, 0x03, 0x73, 0x6e, 0x69, 0x18, 0x13, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x73, + 0x6e, 0x69, 0x42, 0x2e, 0x5a, 0x2c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, + 0x2f, 0x61, 0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x2f, 0x68, 0x69, 0x67, 0x72, 0x65, 0x73, 0x73, + 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x2f, + 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/api/networking/v1/mcp_bridge.proto b/api/networking/v1/mcp_bridge.proto index 8c2c6159d9..53c5fad004 100644 --- a/api/networking/v1/mcp_bridge.proto +++ b/api/networking/v1/mcp_bridge.proto @@ -64,4 +64,6 @@ message RegistryConfig { string consulServiceTag = 15; int64 consulRefreshInterval = 16; string authSecretName = 17; + string protocol = 18; + string sni = 19; } diff --git a/envoy/envoy b/envoy/envoy index 9c9c3b717c..b3541845c1 160000 --- a/envoy/envoy +++ b/envoy/envoy @@ -1 +1 @@ -Subproject commit 9c9c3b717c9a3dd8cb8772ef5de86938aa1c93a8 +Subproject commit b3541845c1a78d817c73806299415439c23488d2 diff --git a/helm/core/Chart.yaml b/helm/core/Chart.yaml index 1a969e7d22..1dca35d253 100644 --- a/helm/core/Chart.yaml +++ b/helm/core/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -appVersion: 2.0.0 +appVersion: 2.0.1 description: Helm chart for deploying higress gateways icon: https://higress.io/img/higress_logo_small.png home: http://higress.io/ @@ -10,4 +10,4 @@ name: higress-core sources: - http://github.com/alibaba/higress type: application -version: 2.0.0 +version: 2.0.1 diff --git a/helm/core/crds/customresourcedefinitions.gen.yaml b/helm/core/crds/customresourcedefinitions.gen.yaml index c6cc49c25b..de7e1b9345 100644 --- a/helm/core/crds/customresourcedefinitions.gen.yaml +++ b/helm/core/crds/customresourcedefinitions.gen.yaml @@ -284,6 +284,10 @@ spec: type: string port: type: integer + protocol: + type: string + sni: + type: string type: type: string zkServicesPath: @@ -302,3 +306,4 @@ spec: subresources: status: {} +--- diff --git a/helm/core/templates/_pod.tpl b/helm/core/templates/_pod.tpl index ce17b31c85..9bc7b744b2 100644 --- a/helm/core/templates/_pod.tpl +++ b/helm/core/templates/_pod.tpl @@ -178,7 +178,7 @@ template: {{- end }} - name: config mountPath: /etc/istio/config - - name: istio-ca-root-cert + - name: higress-ca-root-cert mountPath: /var/run/secrets/istio - name: istio-data mountPath: /var/lib/istio/data @@ -262,7 +262,7 @@ template: expirationSeconds: 43200 path: istio-token {{- end }} - - name: istio-ca-root-cert + - name: higress-ca-root-cert configMap: {{- if .Values.global.enableHigressIstio }} name: istio-ca-root-cert diff --git a/helm/core/templates/configmap.yaml b/helm/core/templates/configmap.yaml index 456f0c521b..2d7ae5b176 100644 --- a/helm/core/templates/configmap.yaml +++ b/helm/core/templates/configmap.yaml @@ -155,7 +155,7 @@ data: "transport_api_version": "V3", "grpc_service": { "envoy_grpc": { - "cluster_name": "service_skywalking" + "cluster_name": "outbound|{{ .Values.tracing.skywalking.port }}||{{ .Values.tracing.skywalking.service }}" } } } @@ -164,36 +164,6 @@ data: {{- end }} "static_resources": { "clusters": [ - {{- if include "skywalking.enabled" . }} - { - "name": "service_skywalking", - "type": "LOGICAL_DNS", - "connect_timeout": "5s", - "http2_protocol_options": { - }, - "dns_lookup_family": "V4_ONLY", - "lb_policy": "ROUND_ROBIN", - "load_assignment": { - "cluster_name": "service_skywalking", - "endpoints": [ - { - "lb_endpoints": [ - { - "endpoint": { - "address": { - "socket_address": { - "address": "{{ .Values.tracing.skywalking.service }}", - "port_value": "{{ .Values.tracing.skywalking.port }}" - } - } - } - } - ] - } - ] - } - }, - {{- end }} { "name": "higress-gateway-local", "type": "STATIC", diff --git a/helm/higress/Chart.lock b/helm/higress/Chart.lock index a8f1278488..e2397ea62a 100644 --- a/helm/higress/Chart.lock +++ b/helm/higress/Chart.lock @@ -1,9 +1,9 @@ dependencies: - name: higress-core repository: file://../core - version: 2.0.0 + version: 2.0.1 - name: higress-console repository: https://higress.io/helm-charts/ - version: 1.4.3 -digest: sha256:ebfedb7faee4973b6e1e3624a9fcc20790943aef76ec60921e0010d1e62ff92a -generated: "2024-09-13T10:36:29.963179+08:00" + version: 1.4.4 +digest: sha256:6e4d77c31c834a404a728ec5a8379dd5df27a7e9b998a08e6524dc6534b07c1d +generated: "2024-10-09T20:07:21.857942+08:00" diff --git a/helm/higress/Chart.yaml b/helm/higress/Chart.yaml index 20c63eee98..7d683863ba 100644 --- a/helm/higress/Chart.yaml +++ b/helm/higress/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -appVersion: 2.0.0 +appVersion: 2.0.1 description: Helm chart for deploying Higress gateways icon: https://higress.io/img/higress_logo_small.png home: http://higress.io/ @@ -12,9 +12,9 @@ sources: dependencies: - name: higress-core repository: "file://../core" - version: 2.0.0 + version: 2.0.1 - name: higress-console repository: "https://higress.io/helm-charts/" - version: 1.4.3 + version: 1.4.4 type: application -version: 2.0.0 +version: 2.0.1 diff --git a/hgctl/pkg/plugin/test/templates.go b/hgctl/pkg/plugin/test/templates.go index 8e1b06ef85..21e596e263 100644 --- a/hgctl/pkg/plugin/test/templates.go +++ b/hgctl/pkg/plugin/test/templates.go @@ -114,6 +114,8 @@ static_resources: value: | {{ .JSONExample }} - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router clusters: - name: httpbin connect_timeout: 30s diff --git a/istio/istio b/istio/istio index 8918eb802a..d380470e53 160000 --- a/istio/istio +++ b/istio/istio @@ -1 +1 @@ -Subproject commit 8918eb802a2ab7aafe91ea5010c0642258d94669 +Subproject commit d380470e53b6aa45b7a8ab2bf26cbc6c147da06f diff --git a/pkg/common/protocol.go b/pkg/common/protocol.go index f3b51626b2..a8af22c43c 100644 --- a/pkg/common/protocol.go +++ b/pkg/common/protocol.go @@ -21,7 +21,10 @@ type Protocol string const ( TCP Protocol = "TCP" HTTP Protocol = "HTTP" + HTTP2 Protocol = "HTTP2" + HTTPS Protocol = "HTTPS" GRPC Protocol = "GRPC" + GRPCS Protocol = "GRPCS" Dubbo Protocol = "Dubbo" Unsupported Protocol = "UnsupportedProtocol" ) @@ -32,8 +35,14 @@ func ParseProtocol(s string) Protocol { return TCP case "http": return HTTP + case "https": + return HTTPS + case "http2": + return HTTP2 case "grpc", "triple", "tri": return GRPC + case "grpcs": + return GRPCS case "dubbo": return Dubbo } @@ -51,7 +60,7 @@ func (p Protocol) IsTCP() bool { func (p Protocol) IsHTTP() bool { switch p { - case HTTP, GRPC: + case HTTP, GRPC, GRPCS, HTTP2, HTTPS: return true default: return false @@ -60,7 +69,16 @@ func (p Protocol) IsHTTP() bool { func (p Protocol) IsGRPC() bool { switch p { - case GRPC: + case GRPC, GRPCS: + return true + default: + return false + } +} + +func (i Protocol) IsHTTPS() bool { + switch i { + case HTTPS, GRPCS: return true default: return false diff --git a/pkg/config/constants/constants.go b/pkg/config/constants/constants.go index 6a24952276..0bde512325 100644 --- a/pkg/config/constants/constants.go +++ b/pkg/config/constants/constants.go @@ -23,3 +23,7 @@ const KnativeIngressCRDName = "ingresses.networking.internal.knative.dev" const KnativeServicesCRDName = "services.serving.knative.dev" const ManagedGatewayController = "higress.io/gateway-controller" + +const RegistryTypeLabelKey = "higress-registry-type" + +const RegistryNameLabelKey = "higress-registry-name" diff --git a/pkg/ingress/config/ingress_config.go b/pkg/ingress/config/ingress_config.go index 2d8b344212..e41181676a 100644 --- a/pkg/ingress/config/ingress_config.go +++ b/pkg/ingress/config/ingress_config.go @@ -53,6 +53,7 @@ import ( extlisterv1 "github.com/alibaba/higress/client/pkg/listers/extensions/v1alpha1" netlisterv1 "github.com/alibaba/higress/client/pkg/listers/networking/v1" "github.com/alibaba/higress/pkg/cert" + higressconst "github.com/alibaba/higress/pkg/config/constants" "github.com/alibaba/higress/pkg/ingress/kube/annotations" "github.com/alibaba/higress/pkg/ingress/kube/common" "github.com/alibaba/higress/pkg/ingress/kube/configmap" @@ -628,8 +629,8 @@ func (m *IngressConfig) convertServiceEntry([]common.WrapperConfig) []config.Con if m.RegistryReconciler == nil { return nil } - serviceEntries := m.RegistryReconciler.GetAllServiceEntryWrapper() - IngressLog.Infof("Found http2rpc serviceEntries %s", serviceEntries) + serviceEntries := m.RegistryReconciler.GetAllServiceWrapper() + IngressLog.Infof("Found mcp serviceEntries %v", serviceEntries) out := make([]config.Config, 0, len(serviceEntries)) for _, se := range serviceEntries { out = append(out, config.Config{ @@ -638,6 +639,10 @@ func (m *IngressConfig) convertServiceEntry([]common.WrapperConfig) []config.Con Name: se.ServiceEntry.Hosts[0], Namespace: "mcp", CreationTimestamp: se.GetCreateTime(), + Labels: map[string]string{ + higressconst.RegistryTypeLabelKey: se.RegistryType, + higressconst.RegistryNameLabelKey: se.RegistryName, + }, }, Spec: se.ServiceEntry, }) @@ -703,6 +708,32 @@ func (m *IngressConfig) convertDestinationRule(configs []common.WrapperConfig) [ destinationRules[serviceName] = dr } + if m.RegistryReconciler != nil { + drws := m.RegistryReconciler.GetAllDestinationRuleWrapper() + IngressLog.Infof("Found mcp destinationRules: %v", drws) + for _, destinationRuleWrapper := range drws { + serviceName := destinationRuleWrapper.ServiceKey.ServiceFQDN + dr, exist := destinationRules[serviceName] + if !exist { + destinationRules[serviceName] = destinationRuleWrapper + } else if dr.DestinationRule.TrafficPolicy != nil { + portTrafficPolicy := destinationRuleWrapper.DestinationRule.TrafficPolicy.PortLevelSettings[0] + portUpdated := false + for _, portTrafficPolicy := range dr.DestinationRule.TrafficPolicy.PortLevelSettings { + if portTrafficPolicy.Port.Number == portTrafficPolicy.Port.Number { + portTrafficPolicy.Tls = portTrafficPolicy.Tls + portUpdated = true + break + } + } + if portUpdated { + continue + } + dr.DestinationRule.TrafficPolicy.PortLevelSettings = append(dr.DestinationRule.TrafficPolicy.PortLevelSettings, portTrafficPolicy) + } + } + } + out := make([]config.Config, 0, len(destinationRules)) for _, dr := range destinationRules { sort.SliceStable(dr.DestinationRule.TrafficPolicy.PortLevelSettings, func(i, j int) bool { @@ -727,6 +758,7 @@ func (m *IngressConfig) convertDestinationRule(configs []common.WrapperConfig) [ Spec: dr.DestinationRule, }) } + return out } @@ -1034,16 +1066,27 @@ func (m *IngressConfig) AddOrUpdateMcpBridge(clusterNamespacedName util.ClusterN } if m.RegistryReconciler == nil { m.RegistryReconciler = reconcile.NewReconciler(func() { - metadata := config.Meta{ + seMetadata := config.Meta{ Name: "mcpbridge-serviceentry", Namespace: m.namespace, GroupVersionKind: gvk.ServiceEntry, // Set this label so that we do not compare configs and just push. Labels: map[string]string{constants.AlwaysPushLabel: "true"}, } + drMetadata := config.Meta{ + Name: "mcpbridge-destinationrule", + Namespace: m.namespace, + GroupVersionKind: gvk.DestinationRule, + // Set this label so that we do not compare configs and just push. + Labels: map[string]string{constants.AlwaysPushLabel: "true"}, + } for _, f := range m.serviceEntryHandlers { IngressLog.Debug("McpBridge triggerd serviceEntry update") - f(config.Config{Meta: metadata}, config.Config{Meta: metadata}, istiomodel.EventUpdate) + f(config.Config{Meta: seMetadata}, config.Config{Meta: seMetadata}, istiomodel.EventUpdate) + } + for _, f := range m.destinationRuleHandlers { + IngressLog.Debug("McpBridge triggerd destinationRule update") + f(config.Config{Meta: drMetadata}, config.Config{Meta: drMetadata}, istiomodel.EventUpdate) } }, m.localKubeClient, m.namespace) } @@ -1489,7 +1532,7 @@ func constructBasicAuthEnvoyFilter(rules *common.BasicAuthRules, namespace strin }, nil } -func QueryByName(serviceEntries []*memory.ServiceEntryWrapper, serviceName string) (*memory.ServiceEntryWrapper, error) { +func QueryByName(serviceEntries []*memory.ServiceWrapper, serviceName string) (*memory.ServiceWrapper, error) { IngressLog.Infof("Found http2rpc serviceEntries %s", serviceEntries) for _, se := range serviceEntries { if se.ServiceName == serviceName { @@ -1499,7 +1542,7 @@ func QueryByName(serviceEntries []*memory.ServiceEntryWrapper, serviceName strin return nil, fmt.Errorf("can't find ServiceEntry by serviceName:%v", serviceName) } -func QueryRpcServiceVersion(serviceEntry *memory.ServiceEntryWrapper, serviceName string) (string, error) { +func QueryRpcServiceVersion(serviceEntry *memory.ServiceWrapper, serviceName string) (string, error) { IngressLog.Infof("Found http2rpc serviceEntry %s", serviceEntry) IngressLog.Infof("Found http2rpc ServiceEntry %s", serviceEntry.ServiceEntry) IngressLog.Infof("Found http2rpc WorkloadSelector %s", serviceEntry.ServiceEntry.WorkloadSelector) diff --git a/pkg/ingress/kube/common/controller.go b/pkg/ingress/kube/common/controller.go index 7db8880f88..be61d9803c 100644 --- a/pkg/ingress/kube/common/controller.go +++ b/pkg/ingress/kube/common/controller.go @@ -52,6 +52,15 @@ type WrapperGateway struct { Host string } +func CreateMcpServiceKey(host string, portNumber int32) ServiceKey { + return ServiceKey{ + Namespace: "mcp", + Name: host, + ServiceFQDN: host, + Port: portNumber, + } +} + func (w *WrapperGateway) IsHTTPS() bool { if w.Gateway == nil || len(w.Gateway.Servers) == 0 { return false diff --git a/pkg/ingress/kube/gateway/controller.go b/pkg/ingress/kube/gateway/controller.go index 9eaa15ddcc..b813053256 100644 --- a/pkg/ingress/kube/gateway/controller.go +++ b/pkg/ingress/kube/gateway/controller.go @@ -22,6 +22,7 @@ import ( kubecredentials "istio.io/istio/pilot/pkg/credentials/kube" "istio.io/istio/pilot/pkg/model" kubecontroller "istio.io/istio/pilot/pkg/serviceregistry/kube/controller" + "istio.io/istio/pilot/pkg/status" "istio.io/istio/pkg/config" "istio.io/istio/pkg/config/constants" "istio.io/istio/pkg/config/schema/collection" @@ -48,6 +49,7 @@ type gatewayController struct { store model.ConfigStoreController credsController credentials.MulticlusterController istioController *istiogateway.Controller + statusManager *status.Manager resourceUpToDate atomic.Bool } @@ -76,9 +78,10 @@ func NewController(client kube.Client, options common.Options) common.GatewayCon istioController.DefaultGatewaySelector = map[string]string{options.GatewaySelectorKey: options.GatewaySelectorValue} } + var statusManager *status.Manager = nil if options.EnableStatus { - // TODO: Add status sync support - //istioController.SetStatusWrite(true,) + statusManager = status.NewManager(store) + istioController.SetStatusWrite(true, statusManager) } else { IngressLog.Infof("Disable status update for cluster %s", clusterId) } @@ -87,6 +90,7 @@ func NewController(client kube.Client, options common.Options) common.GatewayCon store: store, credsController: credsController, istioController: istioController, + statusManager: statusManager, } } @@ -148,6 +152,9 @@ func (g *gatewayController) Run(stop <-chan struct{}) { }) go g.store.Run(stop) go g.istioController.Run(stop) + if g.statusManager != nil { + g.statusManager.Start(stop) + } } func (g *gatewayController) SetWatchErrorHandler(f func(r *cache.Reflector, err error)) error { diff --git a/pkg/ingress/kube/gateway/istio/context.go b/pkg/ingress/kube/gateway/istio/context.go index 53bc97f4d4..86d7230fd3 100644 --- a/pkg/ingress/kube/gateway/istio/context.go +++ b/pkg/ingress/kube/gateway/istio/context.go @@ -15,26 +15,37 @@ package istio import ( + "context" "fmt" "sort" - "strconv" "strings" networking "istio.io/api/networking/v1alpha3" "istio.io/istio/pilot/pkg/model" + serviceRegistryKube "istio.io/istio/pilot/pkg/serviceregistry/kube" "istio.io/istio/pkg/cluster" - "istio.io/istio/pkg/config/host" + "istio.io/istio/pkg/config/schema/gvk" + "istio.io/istio/pkg/kube" "istio.io/istio/pkg/util/sets" corev1 "k8s.io/api/core/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // GatewayContext contains a minimal subset of push context functionality to be exposed to GatewayAPIControllers type GatewayContext struct { ps *model.PushContext + // Start - Updated by Higress + client kube.Client + domainSuffix string + clusterID cluster.ID + // End - Updated by Higress } -func NewGatewayContext(ps *model.PushContext) GatewayContext { - return GatewayContext{ps} +// Start - Updated by Higress + +func NewGatewayContext(ps *model.PushContext, client kube.Client, domainSuffix string, clusterID cluster.ID) GatewayContext { + return GatewayContext{ps, client, domainSuffix, clusterID} } // ResolveGatewayInstances attempts to resolve all instances that a gateway will be exposed on. @@ -59,26 +70,20 @@ func (gc GatewayContext) ResolveGatewayInstances( foundExternal := sets.New[string]() foundPending := sets.New[string]() warnings := []string{} + + // Cache endpoints to reduce redundant queries + endpointsCache := make(map[string]*corev1.Endpoints) + for _, g := range gwsvcs { - svc, f := gc.ps.ServiceIndex.HostnameAndNamespace[host.Name(g)][namespace] - if !f { - otherNamespaces := []string{} - for ns := range gc.ps.ServiceIndex.HostnameAndNamespace[host.Name(g)] { - otherNamespaces = append(otherNamespaces, `"`+ns+`"`) // Wrap in quotes for output - } - if len(otherNamespaces) > 0 { - sort.Strings(otherNamespaces) - warnings = append(warnings, fmt.Sprintf("hostname %q not found in namespace %q, but it was found in namespace(s) %v", - g, namespace, strings.Join(otherNamespaces, ", "))) - } else { - warnings = append(warnings, fmt.Sprintf("hostname %q not found", g)) - } + svc := gc.GetService(g, namespace, gvk.Service.Kind) + if svc == nil { + warnings = append(warnings, fmt.Sprintf("hostname %q not found", g)) continue } - svcKey := svc.Key() + for port := range ports { - instances := gc.ps.ServiceInstancesByPort(svc, port, nil) - if len(instances) > 0 { + exists := checkServicePortExists(svc, port) + if exists { foundInternal.Insert(fmt.Sprintf("%s:%d", g, port)) if svc.Attributes.ClusterExternalAddresses.Len() > 0 { // Fetch external IPs from all clusters @@ -92,22 +97,30 @@ func (gc GatewayContext) ResolveGatewayInstances( } } } else { - instancesByPort := gc.ps.ServiceInstances(svcKey) - if instancesEmpty(instancesByPort) { + endpoints, ok := endpointsCache[g] + if !ok { + endpoints = gc.GetEndpoints(g, namespace) + endpointsCache[g] = endpoints + } + + if endpoints == nil { warnings = append(warnings, fmt.Sprintf("no instances found for hostname %q", g)) } else { - hintPort := sets.New[string]() - for _, instances := range instancesByPort { - for _, i := range instances { - if i.Endpoint.EndpointPort == uint32(port) { - hintPort.Insert(strconv.Itoa(i.ServicePort.Port)) + hintWorkloadPort := false + for _, subset := range endpoints.Subsets { + for _, subSetPort := range subset.Ports { + if subSetPort.Port == int32(port) { + hintWorkloadPort = true + break } } + if hintWorkloadPort { + break + } } - if hintPort.Len() > 0 { + if hintWorkloadPort { warnings = append(warnings, fmt.Sprintf( - "port %d not found for hostname %q (hint: the service port should be specified, not the workload port. Did you mean one of these ports: %v?)", - port, g, sets.SortedList(hintPort))) + "port %d not found for hostname %q (hint: the service port should be specified, not the workload port", port, g)) } else { warnings = append(warnings, fmt.Sprintf("port %d not found for hostname %q", port, g)) } @@ -119,15 +132,60 @@ func (gc GatewayContext) ResolveGatewayInstances( return sets.SortedList(foundInternal), sets.SortedList(foundExternal), sets.SortedList(foundPending), warnings } -func (gc GatewayContext) GetService(hostname, namespace string) *model.Service { - return gc.ps.ServiceIndex.HostnameAndNamespace[host.Name(hostname)][namespace] +func (gc GatewayContext) GetService(hostname, namespace, kind string) *model.Service { + // Currently only supports type Kubernetes Service + if kind != gvk.Service.Kind { + log.Warnf("Unsupported kind: expected 'Service', but got '%s'", kind) + return nil + } + serviceName := extractServiceName(hostname) + + svc, err := gc.client.Kube().CoreV1().Services(namespace).Get(context.TODO(), serviceName, metav1.GetOptions{}) + if err != nil { + if kerrors.IsNotFound(err) { + return nil + } + log.Errorf("failed to get service (serviceName: %s, namespace: %s): %v", serviceName, namespace, err) + return nil + } + + return serviceRegistryKube.ConvertService(*svc, gc.domainSuffix, gc.clusterID) } -func instancesEmpty(m map[int][]*model.ServiceInstance) bool { - for _, instances := range m { - if len(instances) > 0 { - return false +func (gc GatewayContext) GetEndpoints(hostname, namespace string) *corev1.Endpoints { + serviceName := extractServiceName(hostname) + + endpoints, err := gc.client.Kube().CoreV1().Endpoints(namespace).Get(context.TODO(), serviceName, metav1.GetOptions{}) + + if err != nil { + if kerrors.IsNotFound(err) { + return nil } + log.Errorf("failed to get endpoints (serviceName: %s, namespace: %s): %v", serviceName, namespace, err) + return nil } - return true + + return endpoints } + +func checkServicePortExists(svc *model.Service, port int) bool { + if svc == nil { + return false + } + for _, svcPort := range svc.Ports { + if port == svcPort.Port { + return true + } + } + return false +} + +func extractServiceName(hostName string) string { + parts := strings.Split(hostName, ".") + if len(parts) >= 4 { + return parts[0] + } + return "" +} + +// End - Updated by Higress diff --git a/pkg/ingress/kube/gateway/istio/controller.go b/pkg/ingress/kube/gateway/istio/controller.go index 5592701ad4..1b09776871 100644 --- a/pkg/ingress/kube/gateway/istio/controller.go +++ b/pkg/ingress/kube/gateway/istio/controller.go @@ -201,7 +201,9 @@ func (c *Controller) Reconcile(ps *model.PushContext) error { ReferenceGrant: referenceGrant, DefaultGatewaySelector: c.DefaultGatewaySelector, Domain: c.domain, - Context: NewGatewayContext(ps), + // Start - Updated by Higress + Context: NewGatewayContext(ps, c.client, c.domain, c.cluster), + // End - Updated by Higress } if !input.hasResources() { diff --git a/pkg/ingress/kube/gateway/istio/conversion.go b/pkg/ingress/kube/gateway/istio/conversion.go index 1e7bc31597..83c817a717 100644 --- a/pkg/ingress/kube/gateway/istio/conversion.go +++ b/pkg/ingress/kube/gateway/istio/conversion.go @@ -1168,7 +1168,7 @@ func buildDestination(ctx configContext, to k8s.BackendRef, ns string, enforceRe return nil, &ConfigError{Reason: InvalidDestination, Message: "serviceName invalid; the name of the Service must be used, not the hostname."} } hostname := fmt.Sprintf("%s.%s.svc.%s", to.Name, namespace, ctx.Domain) - if ctx.Context.GetService(hostname, namespace) == nil { + if ctx.Context.GetService(hostname, namespace, gvk.Service.Kind) == nil { invalidBackendErr = &ConfigError{Reason: InvalidDestinationNotFound, Message: fmt.Sprintf("backend(%s) not found", hostname)} } return &istio.Destination{ @@ -1192,7 +1192,7 @@ func buildDestination(ctx configContext, to k8s.BackendRef, ns string, enforceRe if strings.Contains(string(to.Name), ".") { return nil, &ConfigError{Reason: InvalidDestination, Message: "serviceName invalid; the name of the Service must be used, not the hostname."} } - if ctx.Context.GetService(hostname, namespace) == nil { + if ctx.Context.GetService(hostname, namespace, "ServiceImport") == nil { invalidBackendErr = &ConfigError{Reason: InvalidDestinationNotFound, Message: fmt.Sprintf("backend(%s) not found", hostname)} } return &istio.Destination{ @@ -1210,7 +1210,7 @@ func buildDestination(ctx configContext, to k8s.BackendRef, ns string, enforceRe return nil, &ConfigError{Reason: InvalidDestination, Message: "namespace may not be set with Hostname type"} } hostname := string(to.Name) - if ctx.Context.GetService(hostname, namespace) == nil { + if ctx.Context.GetService(hostname, namespace, "Hostname") == nil { invalidBackendErr = &ConfigError{Reason: InvalidDestinationNotFound, Message: fmt.Sprintf("backend(%s) not found", hostname)} } return &istio.Destination{ diff --git a/pkg/ingress/kube/gateway/istio/conversion_test.go b/pkg/ingress/kube/gateway/istio/conversion_test.go index 661295a6e1..986aecbc69 100644 --- a/pkg/ingress/kube/gateway/istio/conversion_test.go +++ b/pkg/ingress/kube/gateway/istio/conversion_test.go @@ -17,6 +17,7 @@ package istio import ( + "context" "fmt" "os" "reflect" @@ -25,6 +26,7 @@ import ( "strings" "testing" + "github.com/google/go-cmp/cmp" "istio.io/istio/pilot/pkg/config/kube/crd" credentials "istio.io/istio/pilot/pkg/credentials/kube" "istio.io/istio/pilot/pkg/features" @@ -47,7 +49,8 @@ import ( "sigs.k8s.io/yaml" ) -var ports = []*model.Port{ +// Start - Updated by Higress +var ports = []corev1.ServicePort{ { Name: "http", Port: 80, @@ -64,232 +67,291 @@ var defaultGatewaySelector = map[string]string{ "higress": "higress-system-higress-gateway", } -var services = []*model.Service{ +var services = []corev1.Service{ { - Attributes: model.ServiceAttributes{ + ObjectMeta: metav1.ObjectMeta{ Name: "higress-gateway", Namespace: "higress-system", - ClusterExternalAddresses: &model.AddressMap{ - Addresses: map[cluster.ID][]string{ - "Kubernetes": {"1.2.3.4"}, - }, - }, }, - Ports: ports, - Hostname: "higress-gateway.higress-system.svc.domain.suffix", + Spec: corev1.ServiceSpec{ + Ports: ports, + ExternalIPs: []string{"1.2.3.4"}, + }, }, { - Attributes: model.ServiceAttributes{ + ObjectMeta: metav1.ObjectMeta{ + Name: "example.com", Namespace: "higress-system", }, - Ports: ports, - Hostname: "example.com", + Spec: corev1.ServiceSpec{ + Ports: ports, + }, }, { - Attributes: model.ServiceAttributes{ + ObjectMeta: metav1.ObjectMeta{ + Name: "httpbin", Namespace: "default", }, - Ports: ports, - Hostname: "httpbin.default.svc.domain.suffix", + Spec: corev1.ServiceSpec{ + Ports: ports, + }, }, { - Attributes: model.ServiceAttributes{ + ObjectMeta: metav1.ObjectMeta{ + Name: "httpbin-apple", Namespace: "apple", }, - Ports: ports, - Hostname: "httpbin-apple.apple.svc.domain.suffix", + Spec: corev1.ServiceSpec{ + Ports: ports, + }, }, { - Attributes: model.ServiceAttributes{ + ObjectMeta: metav1.ObjectMeta{ + Name: "httpbin-banana", Namespace: "banana", }, - Ports: ports, - Hostname: "httpbin-banana.banana.svc.domain.suffix", - }, - { - Attributes: model.ServiceAttributes{ - Namespace: "default", + Spec: corev1.ServiceSpec{ + Ports: ports, }, - Ports: ports, - Hostname: "httpbin-second.default.svc.domain.suffix", }, { - Attributes: model.ServiceAttributes{ + ObjectMeta: metav1.ObjectMeta{ + Name: "httpbin-second", Namespace: "default", }, - Ports: ports, - Hostname: "httpbin-wildcard.default.svc.domain.suffix", + Spec: corev1.ServiceSpec{ + Ports: ports, + }, }, { - Attributes: model.ServiceAttributes{ + ObjectMeta: metav1.ObjectMeta{ + Name: "httpbin-wildcard", Namespace: "default", }, - Ports: ports, - Hostname: "foo-svc.default.svc.domain.suffix", + Spec: corev1.ServiceSpec{ + Ports: ports, + }, }, { - Attributes: model.ServiceAttributes{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo-svc", Namespace: "default", }, - Ports: ports, - Hostname: "httpbin-other.default.svc.domain.suffix", + Spec: corev1.ServiceSpec{ + Ports: ports, + }, }, { - Attributes: model.ServiceAttributes{ + ObjectMeta: metav1.ObjectMeta{ + Name: "httpbin-other", Namespace: "default", }, - Ports: ports, - Hostname: "example.default.svc.domain.suffix", + Spec: corev1.ServiceSpec{ + Ports: ports, + }, }, { - Attributes: model.ServiceAttributes{ + ObjectMeta: metav1.ObjectMeta{ + Name: "example", Namespace: "default", }, - Ports: ports, - Hostname: "echo.default.svc.domain.suffix", + Spec: corev1.ServiceSpec{ + Ports: ports, + }, }, { - Attributes: model.ServiceAttributes{ + ObjectMeta: metav1.ObjectMeta{ + Name: "echo", Namespace: "default", }, - Ports: ports, - Hostname: "echo.default.svc.domain.suffix", + Spec: corev1.ServiceSpec{ + Ports: ports, + }, }, { - Attributes: model.ServiceAttributes{ + ObjectMeta: metav1.ObjectMeta{ + Name: "httpbin", Namespace: "cert", }, - Ports: ports, - Hostname: "httpbin.cert.svc.domain.suffix", + Spec: corev1.ServiceSpec{ + Ports: ports, + }, }, { - Attributes: model.ServiceAttributes{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-svc", Namespace: "service", }, - Ports: ports, - Hostname: "my-svc.service.svc.domain.suffix", + Spec: corev1.ServiceSpec{ + Ports: ports, + }, }, { - Attributes: model.ServiceAttributes{ + ObjectMeta: metav1.ObjectMeta{ + Name: "google.com", Namespace: "default", }, - Ports: ports, - Hostname: "google.com", + Spec: corev1.ServiceSpec{ + Ports: ports, + }, }, { - Attributes: model.ServiceAttributes{ + ObjectMeta: metav1.ObjectMeta{ + Name: "svc2", Namespace: "allowed-1", }, - Ports: ports, - Hostname: "svc2.allowed-1.svc.domain.suffix", + Spec: corev1.ServiceSpec{ + Ports: ports, + }, }, { - Attributes: model.ServiceAttributes{ + ObjectMeta: metav1.ObjectMeta{ + Name: "svc2", Namespace: "allowed-2", }, - Ports: ports, - Hostname: "svc2.allowed-2.svc.domain.suffix", + Spec: corev1.ServiceSpec{ + Ports: ports, + }, }, { - Attributes: model.ServiceAttributes{ + ObjectMeta: metav1.ObjectMeta{ + Name: "svc1", Namespace: "allowed-1", }, - Ports: ports, - Hostname: "svc1.allowed-1.svc.domain.suffix", + Spec: corev1.ServiceSpec{ + Ports: ports, + }, }, { - Attributes: model.ServiceAttributes{ + ObjectMeta: metav1.ObjectMeta{ + Name: "svc3", Namespace: "allowed-2", }, - Ports: ports, - Hostname: "svc3.allowed-2.svc.domain.suffix", + Spec: corev1.ServiceSpec{ + Ports: ports, + }, }, { - Attributes: model.ServiceAttributes{ + ObjectMeta: metav1.ObjectMeta{ + Name: "svc4", Namespace: "default", }, - Ports: ports, - Hostname: "svc4.default.svc.domain.suffix", + Spec: corev1.ServiceSpec{ + Ports: ports, + }, }, { - Attributes: model.ServiceAttributes{ + ObjectMeta: metav1.ObjectMeta{ + Name: "httpbin", Namespace: "group-namespace1", }, - Ports: ports, - Hostname: "httpbin.group-namespace1.svc.domain.suffix", + Spec: corev1.ServiceSpec{ + Ports: ports, + }, }, { - Attributes: model.ServiceAttributes{ + ObjectMeta: metav1.ObjectMeta{ + Name: "httpbin", Namespace: "group-namespace2", }, - Ports: ports, - Hostname: "httpbin.group-namespace2.svc.domain.suffix", + Spec: corev1.ServiceSpec{ + Ports: ports, + }, }, { - Attributes: model.ServiceAttributes{ + ObjectMeta: metav1.ObjectMeta{ + Name: "httpbin-zero", Namespace: "default", }, - Ports: ports, - Hostname: "httpbin-zero.default.svc.domain.suffix", + Spec: corev1.ServiceSpec{ + Ports: ports, + }, }, { - Attributes: model.ServiceAttributes{ + ObjectMeta: metav1.ObjectMeta{ + Name: "httpbin", Namespace: "higress-system", }, - Ports: ports, - Hostname: "httpbin.higress-system.svc.domain.suffix", + Spec: corev1.ServiceSpec{ + Ports: ports, + }, }, { - Attributes: model.ServiceAttributes{ + ObjectMeta: metav1.ObjectMeta{ + Name: "httpbin-mirror", Namespace: "default", }, - Ports: ports, - Hostname: "httpbin-mirror.default.svc.domain.suffix", + Spec: corev1.ServiceSpec{ + Ports: ports, + }, }, { - Attributes: model.ServiceAttributes{ + ObjectMeta: metav1.ObjectMeta{ + Name: "httpbin-foo", Namespace: "default", }, - Ports: ports, - Hostname: "httpbin-foo.default.svc.domain.suffix", + Spec: corev1.ServiceSpec{ + Ports: ports, + }, }, { - Attributes: model.ServiceAttributes{ + ObjectMeta: metav1.ObjectMeta{ + Name: "httpbin-alt", Namespace: "default", }, - Ports: ports, - Hostname: "httpbin-alt.default.svc.domain.suffix", + Spec: corev1.ServiceSpec{ + Ports: ports, + }, }, { - Attributes: model.ServiceAttributes{ + ObjectMeta: metav1.ObjectMeta{ + Name: "higress-controller", Namespace: "higress-system", }, - Ports: ports, - Hostname: "higress-controller.higress-system.svc.domain.suffix", + Spec: corev1.ServiceSpec{ + Ports: ports, + }, }, { - Attributes: model.ServiceAttributes{ + ObjectMeta: metav1.ObjectMeta{ + Name: "echo", Namespace: "higress-system", }, - Ports: ports, - Hostname: "higress-controller.higress-system.svc.domain.suffix", + Spec: corev1.ServiceSpec{ + Ports: ports, + }, }, { - Attributes: model.ServiceAttributes{ - Namespace: "higress-system", + ObjectMeta: metav1.ObjectMeta{ + Name: "httpbin-bad", + Namespace: "default", + }, + Spec: corev1.ServiceSpec{ + Ports: ports, }, - Ports: ports, - Hostname: "echo.higress-system.svc.domain.suffix", }, +} + +var endpoints = []corev1.Endpoints{ { - Attributes: model.ServiceAttributes{ - Namespace: "default", + ObjectMeta: metav1.ObjectMeta{ + Name: "higress-gateway", + Namespace: "higress-system", + }, + Subsets: []corev1.EndpointSubset{ + { + Ports: []corev1.EndpointPort{ + { + Port: 8080, + }, + }, + }, }, - Ports: ports, - Hostname: "httpbin-bad.default.svc.domain.suffix", }, } +// End - Updated by Higress + var ( // https://github.com/kubernetes/kubernetes/blob/v1.25.4/staging/src/k8s.io/kubectl/pkg/cmd/create/create_secret_tls_test.go#L31 rsaCertPEM = `-----BEGIN CERTIFICATE----- @@ -364,6 +426,21 @@ func init() { func TestConvertResources(t *testing.T) { validator := crdvalidation.NewIstioValidator(t) + + // Start - Updated by Higress + client := kube.NewFakeClient() + for _, svc := range services { + if _, err := client.Kube().CoreV1().Services(svc.Namespace).Create(context.TODO(), &svc, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + } + for _, endpoint := range endpoints { + if _, err := client.Kube().CoreV1().Endpoints(endpoint.Namespace).Create(context.TODO(), &endpoint, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + } + // End - Updated by Higress + cases := []struct { name string }{ @@ -374,38 +451,23 @@ func TestConvertResources(t *testing.T) { {"weighted"}, {"zero"}, {"invalid"}, - {"multi-gateway"}, + // 目前仅支持 type 为 Hostname 和 ServiceImport + //{"multi-gateway"}, {"delegated"}, {"route-binding"}, {"reference-policy-tls"}, {"reference-policy-service"}, - {"serviceentry"}, + //{"serviceentry"}, {"alias"}, - {"mcs"}, + //{"mcs"}, {"route-precedence"}, } for _, tt := range cases { t.Run(tt.name, func(t *testing.T) { input := readConfig(t, fmt.Sprintf("testdata/%s.yaml", tt.name), validator) - // Setup a few preconfigured services - instances := []*model.ServiceInstance{} - for _, svc := range services { - instances = append(instances, &model.ServiceInstance{ - Service: svc, - ServicePort: ports[0], - Endpoint: &model.IstioEndpoint{EndpointPort: 8080}, - }, &model.ServiceInstance{ - Service: svc, - ServicePort: ports[1], - Endpoint: &model.IstioEndpoint{}, - }) - } - cg := v1alpha3.NewConfigGenTest(t, v1alpha3.TestOptions{ - Services: services, - Instances: instances, - }) + cg := v1alpha3.NewConfigGenTest(t, v1alpha3.TestOptions{}) kr := splitInput(t, input) - kr.Context = NewGatewayContext(cg.PushContext()) + kr.Context = NewGatewayContext(cg.PushContext(), client, "domain.suffix", "") output := convertResources(kr) output.AllowedReferences = AllowedReferences{} // Not tested here output.ReferencedNamespaceKeys = nil // Not tested here @@ -427,20 +489,20 @@ func TestConvertResources(t *testing.T) { assert.Equal(t, golden, output) - //outputStatus := getStatus(t, kr.GatewayClass, kr.Gateway, kr.HTTPRoute, kr.TLSRoute, kr.TCPRoute) - //goldenStatusFile := fmt.Sprintf("testdata/%s.status.yaml.golden", tt.name) - //if util.Refresh() { - // if err := os.WriteFile(goldenStatusFile, outputStatus, 0o644); err != nil { - // t.Fatal(err) - // } - //} - //goldenStatus, err := os.ReadFile(goldenStatusFile) - //if err != nil { - // t.Fatal(err) - //} - //if diff := cmp.Diff(string(goldenStatus), string(outputStatus)); diff != "" { - // t.Fatalf("Diff:\n%s", diff) - //} + outputStatus := getStatus(t, kr.GatewayClass, kr.Gateway, kr.HTTPRoute, kr.TLSRoute, kr.TCPRoute) + goldenStatusFile := fmt.Sprintf("testdata/%s.status.yaml.golden", tt.name) + if util.Refresh() { + if err := os.WriteFile(goldenStatusFile, outputStatus, 0o644); err != nil { + t.Fatal(err) + } + } + goldenStatus, err := os.ReadFile(goldenStatusFile) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(string(goldenStatus), string(outputStatus)); diff != "" { + t.Fatalf("Diff:\n%s", diff) + } }) } } @@ -593,7 +655,7 @@ spec: input := readConfigString(t, tt.config, validator) cg := v1alpha3.NewConfigGenTest(t, v1alpha3.TestOptions{}) kr := splitInput(t, input) - kr.Context = NewGatewayContext(cg.PushContext()) + kr.Context = NewGatewayContext(cg.PushContext(), nil, "", "") output := convertResources(kr) c := &Controller{ state: output, @@ -814,7 +876,7 @@ func BenchmarkBuildHTTPVirtualServices(b *testing.B) { validator := crdvalidation.NewIstioValidator(b) input := readConfig(b, "testdata/benchmark-httproute.yaml", validator) kr := splitInput(b, input) - kr.Context = NewGatewayContext(cg.PushContext()) + kr.Context = NewGatewayContext(cg.PushContext(), nil, "", "") ctx := configContext{ GatewayResources: kr, AllowedReferences: convertReferencePolicies(kr), diff --git a/pkg/ingress/kube/gateway/istio/testdata/invalid.status.yaml.golden b/pkg/ingress/kube/gateway/istio/testdata/invalid.status.yaml.golden index 1642b5f13e..632e80d567 100644 --- a/pkg/ingress/kube/gateway/istio/testdata/invalid.status.yaml.golden +++ b/pkg/ingress/kube/gateway/istio/testdata/invalid.status.yaml.golden @@ -128,8 +128,7 @@ status: - lastTransitionTime: fake message: 'Failed to assign to any requested addresses: port 8080 not found for hostname "higress-gateway.higress-system.svc.domain.suffix" (hint: the service - port should be specified, not the workload port. Did you mean one of these ports: - [80]?)' + port should be specified, not the workload port' reason: Invalid status: "False" type: Programmed @@ -163,26 +162,6 @@ status: --- apiVersion: gateway.networking.k8s.io/v1beta1 kind: Gateway -metadata: - creationTimestamp: null - name: invalid-gateway-address - namespace: invalid-gateway-address -spec: null -status: - conditions: - - lastTransitionTime: fake - message: only Hostname is supported, ignoring [1.2.3.4] - reason: UnsupportedAddress - status: "False" - type: Accepted - - lastTransitionTime: fake - message: Failed to assign to any requested addresses - reason: UnsupportedAddress - status: "False" - type: Programmed ---- -apiVersion: gateway.networking.k8s.io/v1beta1 -kind: Gateway metadata: creationTimestamp: null name: invalid-cert-kind @@ -477,4 +456,29 @@ status: namespace: higress-system sectionName: fake --- - +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: HTTPRoute +metadata: + creationTimestamp: null + name: no-backend + namespace: default +spec: null +status: + parents: + - conditions: + - lastTransitionTime: fake + message: Route was valid + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: fake + message: All references resolved + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + controllerName: higress.io/gateway-controller + parentRef: + group: "" + kind: Service + name: httpbin +--- diff --git a/pkg/ingress/kube/gateway/istio/testdata/invalid.yaml b/pkg/ingress/kube/gateway/istio/testdata/invalid.yaml index cddda93a2f..ac6793640d 100644 --- a/pkg/ingress/kube/gateway/istio/testdata/invalid.yaml +++ b/pkg/ingress/kube/gateway/istio/testdata/invalid.yaml @@ -55,22 +55,23 @@ spec: hostname: "*.example" port: 8080 # Test service has port 80 with targetPort 8080 protocol: HTTP ---- -apiVersion: gateway.networking.k8s.io/v1alpha2 -kind: Gateway -metadata: - name: invalid-gateway-address - namespace: invalid-gateway-address -spec: - gatewayClassName: higress - addresses: - - value: 1.2.3.4 - type: istio.io/FakeType - listeners: - - name: default - hostname: "*.domain.example" - port: 80 - protocol: HTTP +#--- +# Higress 仅支持 addresses type 为 Hostname +#apiVersion: gateway.networking.k8s.io/v1alpha2 +#kind: Gateway +#metadata: +# name: invalid-gateway-address +# namespace: invalid-gateway-address +#spec: +# gatewayClassName: higress +# addresses: +# - value: 1.2.3.4 +# type: istio.io/FakeType +# listeners: +# - name: default +# hostname: "*.domain.example" +# port: 80 +# protocol: HTTP --- apiVersion: gateway.networking.k8s.io/v1alpha2 kind: Gateway diff --git a/pkg/ingress/kube/gateway/istio/testdata/invalid.yaml.golden b/pkg/ingress/kube/gateway/istio/testdata/invalid.yaml.golden index f9b31f8ed0..466e230fdc 100644 --- a/pkg/ingress/kube/gateway/istio/testdata/invalid.yaml.golden +++ b/pkg/ingress/kube/gateway/istio/testdata/invalid.yaml.golden @@ -53,25 +53,6 @@ spec: protocol: HTTP --- apiVersion: networking.istio.io/v1alpha3 -kind: Gateway -metadata: - annotations: - internal.istio.io/parents: Gateway/invalid-gateway-address/default.invalid-gateway-address - creationTimestamp: null - name: invalid-gateway-address-istio-autogenerated-k8s-gateway-default - namespace: invalid-gateway-address -spec: - selector: - higress: higress-system-higress-gateway - servers: - - hosts: - - invalid-gateway-address/*.domain.example - port: - name: default - number: 80 - protocol: HTTP ---- -apiVersion: networking.istio.io/v1alpha3 kind: VirtualService metadata: annotations: diff --git a/pkg/ingress/kube/ingress/controller.go b/pkg/ingress/kube/ingress/controller.go index f55d64c3c0..ddfa2b0054 100644 --- a/pkg/ingress/kube/ingress/controller.go +++ b/pkg/ingress/kube/ingress/controller.go @@ -920,12 +920,7 @@ func (c *controller) storeBackendTrafficPolicy(wrapper *common.WrapperConfig, ba if common.ValidateBackendResource(backend.Resource) && wrapper.AnnotationsConfig.Destination != nil { for _, dest := range wrapper.AnnotationsConfig.Destination.McpDestination { portNumber := dest.Destination.GetPort().GetNumber() - serviceKey := common.ServiceKey{ - Namespace: "mcp", - Name: dest.Destination.Host, - Port: int32(portNumber), - ServiceFQDN: dest.Destination.Host, - } + serviceKey := common.CreateMcpServiceKey(dest.Destination.Host, int32(portNumber)) if _, exist := store[serviceKey]; !exist { if serviceKey.Port != 0 { store[serviceKey] = &common.WrapperTrafficPolicy{ diff --git a/pkg/ingress/kube/ingressv1/controller.go b/pkg/ingress/kube/ingressv1/controller.go index 7e493e9f57..847f2eb852 100644 --- a/pkg/ingress/kube/ingressv1/controller.go +++ b/pkg/ingress/kube/ingressv1/controller.go @@ -900,12 +900,7 @@ func (c *controller) storeBackendTrafficPolicy(wrapper *common.WrapperConfig, ba if common.ValidateBackendResource(backend.Resource) && wrapper.AnnotationsConfig.Destination != nil { for _, dest := range wrapper.AnnotationsConfig.Destination.McpDestination { portNumber := dest.Destination.GetPort().GetNumber() - serviceKey := common.ServiceKey{ - Namespace: "mcp", - Name: dest.Destination.Host, - Port: int32(portNumber), - ServiceFQDN: dest.Destination.Host, - } + serviceKey := common.CreateMcpServiceKey(dest.Destination.Host, int32(portNumber)) if _, exist := store[serviceKey]; !exist { if serviceKey.Port != 0 { store[serviceKey] = &common.WrapperTrafficPolicy{ diff --git a/pkg/ingress/mcp/generator.go b/pkg/ingress/mcp/generator.go index 0fb6f7aad4..693a8c923f 100644 --- a/pkg/ingress/mcp/generator.go +++ b/pkg/ingress/mcp/generator.go @@ -64,7 +64,7 @@ func (c ServiceEntryGenerator) Generate(proxy *model.Proxy, w *model.WatchedReso return serviceEntries[i].CreationTimestamp.Before(serviceEntries[j].CreationTimestamp) }) } - return generate(proxy, serviceEntries, w, updates, false, false) + return generate(proxy, serviceEntries, w, updates, c.GeneratorOptions.KeepConfigLabels, c.GeneratorOptions.KeepConfigAnnotations) } func (c ServiceEntryGenerator) GenerateDeltas(proxy *model.Proxy, updates *model.PushRequest, @@ -82,7 +82,7 @@ type VirtualServiceGenerator struct { func (c VirtualServiceGenerator) Generate(proxy *model.Proxy, w *model.WatchedResource, updates *model.PushRequest) (model.Resources, model.XdsLogDetails, error) { virtualServices := c.Environment.List(gvk.VirtualService, model.NamespaceAll) - return generate(proxy, virtualServices, w, updates, false, false) + return generate(proxy, virtualServices, w, updates, c.GeneratorOptions.KeepConfigLabels, c.GeneratorOptions.KeepConfigAnnotations) } func (c VirtualServiceGenerator) GenerateDeltas(proxy *model.Proxy, updates *model.PushRequest, @@ -100,7 +100,7 @@ type DestinationRuleGenerator struct { func (c DestinationRuleGenerator) Generate(proxy *model.Proxy, w *model.WatchedResource, updates *model.PushRequest) (model.Resources, model.XdsLogDetails, error) { rules := c.Environment.List(gvk.DestinationRule, model.NamespaceAll) - return generate(proxy, rules, w, updates, false, false) + return generate(proxy, rules, w, updates, c.GeneratorOptions.KeepConfigLabels, c.GeneratorOptions.KeepConfigAnnotations) } func (c DestinationRuleGenerator) GenerateDeltas(proxy *model.Proxy, updates *model.PushRequest, @@ -118,7 +118,7 @@ type EnvoyFilterGenerator struct { func (c EnvoyFilterGenerator) Generate(proxy *model.Proxy, w *model.WatchedResource, updates *model.PushRequest) (model.Resources, model.XdsLogDetails, error) { filters := c.Environment.List(gvk.EnvoyFilter, model.NamespaceAll) - return generate(proxy, filters, w, updates, false, false) + return generate(proxy, filters, w, updates, c.GeneratorOptions.KeepConfigLabels, c.GeneratorOptions.KeepConfigAnnotations) } func (c EnvoyFilterGenerator) GenerateDeltas(proxy *model.Proxy, updates *model.PushRequest, @@ -154,7 +154,7 @@ type WasmPluginGenerator struct { func (c WasmPluginGenerator) Generate(proxy *model.Proxy, w *model.WatchedResource, updates *model.PushRequest) (model.Resources, model.XdsLogDetails, error) { wasmPlugins := c.Environment.List(gvk.WasmPlugin, model.NamespaceAll) - return generate(proxy, wasmPlugins, w, updates, false, false) + return generate(proxy, wasmPlugins, w, updates, c.GeneratorOptions.KeepConfigLabels, c.GeneratorOptions.KeepConfigAnnotations) } func (c WasmPluginGenerator) GenerateDeltas(proxy *model.Proxy, push *model.PushContext, updates *model.PushRequest, diff --git a/plugins/wasm-go/extensions/api-workflow/Dockerfile b/plugins/wasm-go/extensions/api-workflow/Dockerfile new file mode 100644 index 0000000000..9b084e0596 --- /dev/null +++ b/plugins/wasm-go/extensions/api-workflow/Dockerfile @@ -0,0 +1,2 @@ +FROM scratch +COPY main.wasm plugin.wasm \ No newline at end of file diff --git a/plugins/wasm-go/extensions/api-workflow/README.md b/plugins/wasm-go/extensions/api-workflow/README.md new file mode 100644 index 0000000000..9819b36884 --- /dev/null +++ b/plugins/wasm-go/extensions/api-workflow/README.md @@ -0,0 +1,384 @@ +--- +title: API 工作流 +keywords: [ API工作流 ] +description: API 工作流插件配置参考 +--- +## 功能说明 +`api工作流 `实现了可编排的API workflow 插件，支持根据配置定义生成DAG并执行工作流 + +## 配置说明 + +| 名称 | 数据类型 | 填写要求 | 默认值 | 描述 | 备注 | +|----------|--------|------| --- |--------|----| +| workflow | object | 必填 | | DAG的定义 | | +| env | object | 选填 | | 一些环境变量 | | + +`env`object的配置字段说明如下： + +| 名称 | 数据类型 | 填写要求 | 默认值 | 描述 | 备注 | +|----------|--------|------|------|-----------|--| +| timeout | int | 选填 | 5000 | 每次请求的过期时间 | 单位是毫秒(ms) | +| max_depth | int | 选填 | 100 | 工作流最大迭代次数 | | + + +`workflow`object的配置字段说明如下： + +| 名称 | 数据类型 | 填写要求 | 默认值 | 描述 | 备注 | +|-------|----------------------| ---- | --- |-----------|----| +| nodes | array of node object | 选填 | | DAG的定义的节点 | | +| edges | array of edge object | 必填 | | DAG的定义的边 | | + +`edge` object的配置字段说明如下： + +| 名称 | 数据类型 | 填写要求 | 默认值 | 描述 | +|-------------| ------ | ---- | --- |------------------------------------------------| +| source | string | 必填 | - | 上一步的操作，必须是定义的node的name，或者初始化工作流的start | +| target | string | 必填 | - | 当前的操作，必须是定义的node的name，或者结束工作流的关键字 end continue | | +| conditional | string | 选填 | - | 这一步是否执行的判断条件 | + +`node` object的配置字段说明如下： + +| 名称 | 数据类型 | 填写要求 | 默认值 | 描述 | 备注 | +| --------------- |------------------------------------|---| --- |-------------------------------|-------------------------------| +| name | string | 必填 | - | node名称 | 全局唯一 | +| service_name | string | 必填 | - | higress配置的服务名称 | | +| service_port | int | 选填 | 80 | higress配置的服务端口 | | +| service_domain | string | 选填 | | higress配置的服务domain | | +| service_path | string | 必填 | | 请求的path | | +| service_headers | array of header object | 选填 | | 请求的头 | | +| service_body_replace_keys| array of bodyReplaceKeyPair object | 选填| 请求body模板替换键值对 | 用来构造请求| 如果为空，则直接使用service_body_tmpl请求 | +| service_body_tmpl | string | 选填 | | 请求的body模板 | | +| service_method | string | 必填 | | 请求的方法 | GET，POST | + +`header` object 的配置字段说明如下： + +| 名称 | 数据类型 | 填写要求 | 默认值 | 描述 | 备注 | +|-------|------------------------|---| --- |-----------| --------- | +| key | string | 必填 | - | 头文件的key | | +| value | string | 必填 | - | 头文件的value | | + +`bodyReplaceKeyPair` object 配置说明 + +| 名称 | 数据类型 | 填写要求 | 默认值 | 描述 | 备注 | +|------|------------------------|---| --- |-----------|--| +| from | string | 必填 | - | 描述数据从哪获得 | | +| to | string | 必填 | - | 描述数据最后放到那 | | + + + +## 用法示例 + +我们把工作流抽象成DAG配置文件，加上控制流和数据流更方便的控制流程和构造请求。 + +![img](img/img.png) + + + +### DAG的定义 + +#### 边edge +描述操作如何编排 + +样例 +```yaml + edges: + - source: start + target: A + - source: start + target: B + - source: start + target: C + - source: A + target: D + - source: B + target: D + - source: C + target: D + - source: D + target: end + conditional: "gt {{D||check}} 0.9" + - source: D + target: E + conditional: "lt {{D||check}} 0.9" + - source: E + target: end +``` +#### 控制流 conditional 和 target +##### 分支 conditional +插件执行到conditional的定义不为空的步骤`edge`时，会根据表达式定义判断这步是否执行，如果判断为否，会跳过这个分支。 +表达式可使用参数，用{{xxx}}标注，具体定义见数据流`模板和变量` +支持比较表达式和例子如下： +`eq arg1 arg2`： arg1 == arg2时为true 不只是数字，支持string +`lt arg1 arg2`： arg1 < arg2时为true +`le arg1 arg2`： arg1 <= arg2时为true +`gt arg1 arg2`： arg1 > arg2时为true +`ge arg1 arg2`： arg1 >= arg2时为true +`and arg1 arg2`: arg1 && arg2 +`or arg1 arg2`: arg1 || arg2 +`contain arg1 arg2`: arg1 包含 arg2时为true +支持and 和 or的嵌套 比如 `and (eq 1 1) (or (contain hello hi) (lt 1 2))` + +##### 结束和执行工作流 target +当target为`name`,执行name的操作 +当target 为`end`，直接返回source的结果，结束工作流 +当target 为`continue`，结束工作流，将请求放行到下一个plugin + +#### 数据流 + +进入plugin的数据（request body）,会根据构造模板json`node.service_body_tmpl`和`node.service_body_replace_keys`构造请求body，并把执行后结果存在key为`nodeName`的上下文里，只支持json格式的数据。 + +##### 模板和变量 +在工作流的配置文件中 +###### edge.conditional +配置文件的定义中，`edge.conditional` 支持模板和变量，方便根据数据流的数据来构建请求数据 +在模板里使用变量来代表数据和过滤。变量使用`{{str1||str2}}`包裹，使用`||`分隔，str1代表使用那个node的输出数据，str2代表如何取数据，过滤表达式基于 [GJSON PATH](https://github.com/tidwall/gjson/blob/master/SYNTAX.md) 语法提取字符串，`@all`代表全都要 + +例子 +```yaml +conditional: "lt {{D||check}} 0.9" +``` +node D 的返回值是 +```json +{"check": 0.99} +``` +解析后的表达式 `lt 0.99 0.9` + +###### node.service_body_tmpl 和 node.service_body_replace_keys +这组配置用来构造请求body，`node.service_body_tmpl`是模板json ，`node.service_body_replace_keys`用来描述如何填充模板json，是一个object的数组，from标识数据从哪里来，to表示填充的位置 +`from`是使用`str1||str2`的字符串，str1代表使用那个node的执行返回数据，str2代表如何取数据，表达式基于 [GJSON PATH](https://github.com/tidwall/gjson/blob/master/SYNTAX.md) 语法提取字符串 +`to`标识数据放哪,表达式基于 [GJSON PATH](https://github.com/tidwall/gjson/blob/master/SYNTAX.md) 语法来描述填充位置，使用的是sjson来拼接json，填充到`tool.service_body_tmpl` 的模板json里 +当`node.service_body_replace_keys`为空时，代表直接发送`node.service_body_tmpl` + +例子 +```yaml + service_body_tmpl: + embeddings: + result: "" + msg: "" + sk: "sk-xxxxxx" + service_body_replace_keys: + - to "embeddings.result" + from "A||output.embeddings.0.embedding" + - to "msg" + from "B||@all" +``` +`A`节点的输出是 +```json +{"embeddings": {"output":{"embeddings":[{"embedding":[0.014398524595686043],"text_index":0}]},"usage":{"total_tokens":12},"request_id":"2a5229bc-53d9-91ca-bce2-00ae5e01a1d3"}} +``` +`B`节点的输出是 +```json +["higress项目主仓库的github地址是什么"] +``` +根据 service_body_tmpl 和 service_body_replace_keys 构造的request body如下 +```json +{"embeddings":{"result":"[0.014398524595686043，......]"},"msg":["higress项目主仓库的github地址是什么"],"sk":"sk-xxxxxx"} +``` + + + +### node的定义 + +具体执行的单元，封装了httpCall，提供http的访问能力，获取各种api的能力。request body支持自主构建。 + +样例 +```yaml + nodes: + - name: "A" + service_domain: "dashscope.aliyuncs.com" + service_name: "dashscope" + service_port: 443 + service_path: "/api/v1/services/embeddings/text-embedding/text-embedding" + service_method: "POST" + service_body_tmpl: + model: "text-embedding-v2" + input: + texts: "" + parameters: + text_type: "query" + service_body_replace_keys: + - from: "start||messages.#(role==user)#.content" + to: "input.texts" + service_headers: + - key: "Authorization" + value: "Bearer sk-b98f462xxxxxxxx" + - key: "Content-Type" + value: "application/json" +``` +这是请求官方 text-embedding-v2模型的请求样例 具体请求可以看 https://help.aliyun.com/zh/dashscope/developer-reference/text-embedding-api-details?spm=a2c22.12281978.0.0.4d596ea2lRn8xW +### 一个工作流的例子 +从三个节点ABC获取信息，等到数据都就位了，再执行D。 并根据D的输出判断是否需要执行E还是直接结束 +![dag.png](img/dag.png) +start的返回值(请求plugin的body) +```json +{ + "model":"qwen-7b-chat-xft", + "frequency_penalty":0, + "max_tokens":800, + "stream":false, + "messages": [{"role":"user","content":"higress项目主仓库的github地址是什么"}], + "presence_penalty":0,"temperature":0.7,"top_p":0.95 +} +``` +A的返回值是 +```json +{ + "output":{ + "embeddings": [ + { + "text_index": 0, + "embedding": [-0.006929283495992422,-0.005336422007530928] + }, + { + "text_index": 1, + "embedding": [-0.006929283495992422,-0.005336422007530928] + }, + { + "text_index": 2, + "embedding": [-0.006929283495992422,-0.005336422007530928] + }, + { + "text_index": 3, + "embedding": [-0.006929283495992422,-0.005336422007530928] + } + ] + }, + "usage":{ + "total_tokens":12 + }, + "request_id":"d89c06fb-46a1-47b6-acb9-bfb17f814969" +} +``` +B的返回值是 +```json +{"llm":"this is b"} +``` +C的返回值是 +```json +{ + "get": "this is c" +} +``` +D的返回值是 +```json +{"check": 0.99, "llm":{}} +``` +E的返回值是 +```json +{"save": "ok", "date":{}} +``` +这个工作流的配置文件如下： +```yaml +env: + max_depth: 100 + timeout: 3000 +workflow: + edges: + - source: start + target: A + - source: start + target: B + - source: start + target: C + - source: A + target: D + - source: B + target: D + - source: C + target: D + - source: D + target: end + conditional: "lt {{D||check}} 0.9" + - source: D + target: E + conditional: "gt {{D||check}} 0.9" + - source: E + target: end + nodes: + - name: "A" + service_domain: "dashscope.aliyuncs.com" + service_name: "dashscope" + service_port: 443 + service_path: "/api/v1/services/embeddings/text-embedding/text-embedding" + service_method: "POST" + service_body_tmpl: + model: "text-embedding-v2" + input: + texts: "" + parameters: + text_type: "query" + service_body_replace_keys: + - from: "start||messages.#(role==user)#.content" + to: "input.texts" + service_headers: + - key: "Authorization" + value: "Bearer sk-b98f462xxxxxxxx" + - key: "Content-Type" + value: "application/json" + - name: "B" + service_body_tmpl: + embeddings: "default" + msg: "default request body" + sk: "sk-xxxxxx" + service_body_replace_keys: + service_headers: + - key: "AK" + value: "ak-xxxxxxxxxxxxxxxxxxxx" + - key: "Content-Type" + value: "application/json" + service_method: "POST" + service_name: "whoai.static" + service_path: "/llm" + service_port: 80 + - name: "C" + service_method: "GET" + service_name: "whoai.static" + service_path: "/get" + service_port: 80 + - name: "D" + service_headers: + service_method: "POST" + service_name: "whoai.static" + service_path: "/check_cache" + service_port: 80 + service_body_tmpl: + A_result: "" + B_result: "" + C_result: "" + service_body_replace_keys: + - from: "A||output.embeddings.0.embedding.0" + to: "A_result" + - from: "B||llm" + to: "B_result" + - from: "C||get" + to: "C_result" + - name: "E" + service_method: "POST" + service_name: "whoai.static" + service_path: "/save_cache" + service_port: 80 + service_body_tmpl: + save: "" + service_body_replace_keys: + - from: "D||llm" + to: "save" +``` +执行请求 +```bash +curl -v '127.0.0.1:8080' -H 'Accept: application/json, text/event-stream' -H 'Content-Type: application/json'--data-raw '{"model":"qwen-7b-chat-xft","frequency_penalty":0,"max_tokens":800,"stream":false,"messages":[{"role":"user","content":"higress项目主仓库的github地址是什么"}],"presence_penalty":0,"temperature":0.7,"top_p":0.95}' +``` + +执行后的简略debug日志,可以看到工作流等到前置的ABC流程执行完毕后，根据返回值构建了D的body` {"A_result":0.007155838584362588,"B_result":"this is b","C_result":"this is c"}`;执行D后，根据D的返回值`{"check": 0.99, "llm":{}}`进行条件判断，最终继续执行了E`gt 0.99 0.9`，然后结束流程 +```bash +[api-workflow] workflow exec task,source is start,target is A, body is {"input":{"texts":["higress项目主仓库的github地址是什么"]},"model":"text-embedding-v2","parameters":{"text_type":"query"}},header is [[Authorization Bearer sk-b98f4628125xxxxxxxxxxxxxxxx] [Content-Type application/json]] +[api-workflow] workflow exec task,source is start,target is B, body is {"embeddings":"default","msg":"default request body","sk":"sk-xxxxxx"},header is [[AK ak-xxxxxxxxxxxxxxxxxxxx] [Content-Type application/json]] +[api-workflow] workflow exec task,source is start,target is C, body is ,header is [] +[api-workflow] source is B,target is D,stauts is map[A:0 B:0 C:0 D:2 E:1] +[api-workflow] source is C,target is D,stauts is map[A:0 B:0 C:0 D:1 E:1] +[api-workflow] source is A,target is D,stauts is map[A:0 B:0 C:0 D:0 E:1] +[api-workflow] workflow exec task,source is A,target is D, body is,header is [] +[api-workflow] source is D,target is end,workflow is pass +[api-workflow] source is D,target is E,stauts is map[A:0 B:0 C:0 D:0 E:0] +[api-workflow] workflow exec task,source is D,target is E, body is {"save":"{\"A_result\":0.007155838584362588,\"B_result\":\"this is b\",\"C_result\":\"this is c\"}"},header is [] +[api-workflow] source is E,target is end,workflow is end +``` diff --git a/plugins/wasm-go/extensions/api-workflow/go.mod b/plugins/wasm-go/extensions/api-workflow/go.mod new file mode 100644 index 0000000000..c3073a8c8b --- /dev/null +++ b/plugins/wasm-go/extensions/api-workflow/go.mod @@ -0,0 +1,21 @@ +module api-workflow + +go 1.19 + +replace github.com/alibaba/higress/plugins/wasm-go => ../.. + +require ( + github.com/alibaba/higress/plugins/wasm-go v0.0.0 + github.com/higress-group/proxy-wasm-go-sdk v0.0.0-20240711023527-ba358c48772f + github.com/tidwall/gjson v1.14.3 + github.com/tidwall/sjson v1.2.5 +) + +require ( + github.com/google/uuid v1.3.0 // indirect + github.com/higress-group/nottinygc v0.0.0-20231101025119-e93c4c2f8520 // indirect + github.com/magefile/mage v1.14.0 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.0 // indirect + github.com/tidwall/resp v0.1.1 // indirect +) diff --git a/plugins/wasm-go/extensions/api-workflow/go.sum b/plugins/wasm-go/extensions/api-workflow/go.sum new file mode 100644 index 0000000000..2995e01db7 --- /dev/null +++ b/plugins/wasm-go/extensions/api-workflow/go.sum @@ -0,0 +1,23 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/higress-group/nottinygc v0.0.0-20231101025119-e93c4c2f8520 h1:IHDghbGQ2DTIXHBHxWfqCYQW1fKjyJ/I7W1pMyUDeEA= +github.com/higress-group/nottinygc v0.0.0-20231101025119-e93c4c2f8520/go.mod h1:Nz8ORLaFiLWotg6GeKlJMhv8cci8mM43uEnLA5t8iew= +github.com/higress-group/proxy-wasm-go-sdk v0.0.0-20240711023527-ba358c48772f h1:ZIiIBRvIw62gA5MJhuwp1+2wWbqL9IGElQ499rUsYYg= +github.com/higress-group/proxy-wasm-go-sdk v0.0.0-20240711023527-ba358c48772f/go.mod h1:hNFjhrLUIq+kJ9bOcs8QtiplSQ61GZXtd2xHKx4BYRo= +github.com/magefile/mage v1.14.0 h1:6QDX3g6z1YvJ4olPhT1wksUcSa/V0a1B+pJb73fBjyo= +github.com/magefile/mage v1.14.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.14.3 h1:9jvXn7olKEHU1S9vwoMGliaT8jq1vJ7IH/n9zD9Dnlw= +github.com/tidwall/gjson v1.14.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/resp v0.1.1 h1:Ly20wkhqKTmDUPlyM1S7pWo5kk0tDu8OoC/vFArXmwE= +github.com/tidwall/resp v0.1.1/go.mod h1:3/FrruOBAxPTPtundW0VXgmsQ4ZBA0Aw714lVYgwFa0= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/plugins/wasm-go/extensions/api-workflow/img/dag.png b/plugins/wasm-go/extensions/api-workflow/img/dag.png new file mode 100644 index 0000000000..92a36a9f7e Binary files /dev/null and b/plugins/wasm-go/extensions/api-workflow/img/dag.png differ diff --git a/plugins/wasm-go/extensions/api-workflow/img/img.png b/plugins/wasm-go/extensions/api-workflow/img/img.png new file mode 100644 index 0000000000..fcb2c659d5 Binary files /dev/null and b/plugins/wasm-go/extensions/api-workflow/img/img.png differ diff --git a/plugins/wasm-go/extensions/api-workflow/main.go b/plugins/wasm-go/extensions/api-workflow/main.go new file mode 100644 index 0000000000..5a4254b8d5 --- /dev/null +++ b/plugins/wasm-go/extensions/api-workflow/main.go @@ -0,0 +1,307 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + ejson "encoding/json" + "errors" + "fmt" + "net/http" + "strings" + + "api-workflow/utils" + . "api-workflow/workflow" + + "github.com/alibaba/higress/plugins/wasm-go/pkg/wrapper" + "github.com/higress-group/proxy-wasm-go-sdk/proxywasm" + "github.com/higress-group/proxy-wasm-go-sdk/proxywasm/types" + "github.com/tidwall/gjson" +) + +const ( + DefaultMaxDepth uint32 = 100 + WorkflowExecStatus string = "workflowExecStatus" + DefaultTimeout uint32 = 5000 +) + +func main() { + wrapper.SetCtx( + "api-workflow", + wrapper.ParseConfigBy(parseConfig), + wrapper.ProcessRequestBodyBy(onHttpRequestBody), + ) +} + +func parseConfig(json gjson.Result, c *PluginConfig, log wrapper.Log) error { + + edges := make([]Edge, 0) + nodes := make(map[string]Node) + var err error + // env + env := json.Get("env") + // timeout + c.Env.Timeout = uint32(env.Get("timeout").Int()) + if c.Env.Timeout == 0 { + c.Env.Timeout = DefaultTimeout + } + // max_depth + c.Env.MaxDepth = uint32(env.Get("max_depth").Int()) + if c.Env.MaxDepth == 0 { + c.Env.MaxDepth = DefaultMaxDepth + } + // workflow + workflow := json.Get("workflow") + if !workflow.Exists() { + return errors.New("workflow is empty") + } + // workflow.edges + edges_ := workflow.Get("edges") + if edges_.Exists() && edges_.IsArray() { + for _, w := range edges_.Array() { + task := Task{} + edge := Edge{} + edge.Source = w.Get("source").String() + if edge.Source == "" { + return errors.New("source is empty") + } + edge.Target = w.Get("target").String() + if edge.Target == "" { + return errors.New("target is empty") + } + edge.Task = &task + + edge.Conditional = w.Get("conditional").String() + edges = append(edges, edge) + } + } + c.Workflow.Edges = edges + // workflow.nodes + nodes_ := workflow.Get("nodes") + if nodes_.Exists() && nodes_.IsArray() { + for _, value := range nodes_.Array() { + node := Node{} + node.Name = value.Get("name").String() + if node.Name == "" { + return errors.New("tool name is empty") + } + node.ServiceName = value.Get("service_name").String() + if node.ServiceName == "" { + return errors.New("tool service name is empty") + } + node.ServicePort = value.Get("service_port").Int() + if node.ServicePort == 0 { + if strings.HasSuffix(node.ServiceName, ".static") { + // use default logic port which is 80 for static service + node.ServicePort = 80 + } else { + return errors.New("tool service port is empty") + } + + } + node.ServiceDomain = value.Get("service_domain").String() + node.ServicePath = value.Get("service_path").String() + if node.ServicePath == "" { + node.ServicePath = "/" + } + node.ServiceMethod = value.Get("service_method").String() + if node.ServiceMethod == "" { + return errors.New("service_method is empty") + } + serviceHeaders := value.Get("service_headers") + if serviceHeaders.Exists() && serviceHeaders.IsArray() { + serviceHeaders_ := []ServiceHeader{} + err = ejson.Unmarshal([]byte(serviceHeaders.Raw), &serviceHeaders_) + node.ServiceHeaders = serviceHeaders_ + } + + node.ServiceBodyTmpl = value.Get("service_body_tmpl").String() + serviceBodyReplaceKeys := value.Get("service_body_replace_keys") + if serviceBodyReplaceKeys.Exists() && serviceBodyReplaceKeys.IsArray() { + serviceBodyReplaceKeys_ := []BodyReplaceKeyPair{} + err = ejson.Unmarshal([]byte(serviceBodyReplaceKeys.Raw), &serviceBodyReplaceKeys_) + node.ServiceBodyReplaceKeys = serviceBodyReplaceKeys_ + if err != nil { + return fmt.Errorf("unmarshal service body replace keys failed, err:%v", err) + } + } + + nodes[node.Name] = node + } + c.Workflow.Nodes = nodes + // workflow.WorkflowExecStatus + c.Workflow.WorkflowExecStatus, err = initWorkflowExecStatus(c) + log.Debugf("init status : %v", c.Workflow.WorkflowExecStatus) + if err != nil { + log.Errorf("init workflow exec status failed, err:%v", err) + return fmt.Errorf("init workflow exec status failed, err:%v", err) + } + } + log.Debugf("config : %v", c) + return nil +} + +func initWorkflowExecStatus(config *PluginConfig) (map[string]int, error) { + result := make(map[string]int) + + for name, _ := range config.Workflow.Nodes { + result[name] = 0 + } + for _, edge := range config.Workflow.Edges { + + if edge.Source == TaskStart || edge.Target == TaskContinue || edge.Target == TaskEnd { + continue + } + + count, ok := result[edge.Target] + if !ok { + return nil, fmt.Errorf("Target %s is not exist in nodes", edge.Target) + } + result[edge.Target] = count + 1 + + } + return result, nil +} + +func onHttpRequestBody(ctx wrapper.HttpContext, config PluginConfig, body []byte, log wrapper.Log) types.Action { + + initHeader := make([][2]string, 0) + // 初始化运行状态 + ctx.SetContext(WorkflowExecStatus, config.Workflow.WorkflowExecStatus) + + // 执行工作流 + for _, edge := range config.Workflow.Edges { + + if edge.Source == TaskStart { + ctx.SetContext(fmt.Sprintf("%s", TaskStart), body) + err := recursive(edge, initHeader, body, 1, config, log, ctx) + if err != nil { + // 工作流处理错误，返回500给用户 + log.Errorf("recursive failed: %v", err) + _ = utils.SendResponse(500, "api-workflow.recursive_failed", utils.MimeTypeTextPlain, fmt.Sprintf("workflow plugin recursive failed: %v", err)) + + } + } + } + + return types.ActionPause +} + +// 放入符合条件的edge +func recursive(edge Edge, headers [][2]string, body []byte, depth uint32, config PluginConfig, log wrapper.Log, ctx wrapper.HttpContext) error { + + var err error + // 防止递归次数太多 + if depth > config.Env.MaxDepth { + return fmt.Errorf("maximum recursion depth reached") + } + + // 判断是不是end + if edge.IsEnd() { + log.Debugf("source is %s,target is %s,workflow is end", edge.Source, edge.Target) + log.Debugf("body is %s", string(body)) + _ = proxywasm.SendHttpResponse(200, headers, body, -1) + return nil + } + // 判断是不是continue + if edge.IsContinue() { + log.Debugf("source is %s,target is %s,workflow is continue", edge.Source, edge.Target) + _ = proxywasm.ResumeHttpRequest() + return nil + } + + // 封装task + err = edge.WrapperTask(config, ctx) + if err != nil { + log.Errorf("workflow exec wrapperTask find error,source is %s,target is %s,error is %v ", edge.Source, edge.Target, err) + return fmt.Errorf("workflow exec wrapperTask find error,source is %s,target is %s,error is %v ", edge.Source, edge.Target, err) + } + + // 执行task + log.Debugf("workflow exec task,source is %s,target is %s, body is %s,header is %v", edge.Source, edge.Target, string(edge.Task.Body), edge.Task.Headers) + err = wrapper.HttpCall(edge.Task.Cluster, edge.Task.Method, edge.Task.ServicePath, edge.Task.Headers, edge.Task.Body, func(statusCode int, responseHeaders http.Header, responseBody []byte) { + log.Debugf("code:%d", statusCode) + // 判断response code + if statusCode < 400 { + + // 存入这轮返回的body + ctx.SetContext(fmt.Sprintf("%s", edge.Target), responseBody) + + headers_ := make([][2]string, len(responseHeaders)) + for key, value := range responseHeaders { + headers_ = append(headers_, [2]string{key, value[0]}) + } + // 判断是否进入下一步 + nextStatus := ctx.GetContext(WorkflowExecStatus).(map[string]int) + + // 进入下一步 + for _, next := range config.Workflow.Edges { + if next.Source == edge.Target { + // 更新workflow status + if next.Target != TaskContinue && next.Target != TaskEnd { + + nextStatus[next.Target] = nextStatus[next.Target] - 1 + log.Debugf("source is %s,target is %s,stauts is %v", next.Source, next.Target, nextStatus) + // 还有没执行完的边 + if nextStatus[next.Target] > 0 { + ctx.SetContext(WorkflowExecStatus, nextStatus) + return + } + // 执行出了问题 + if nextStatus[next.Target] < 0 { + log.Errorf("workflow exec status find error %v", nextStatus) + _ = utils.SendResponse(500, "api-workflow.exec_task_failed", utils.MimeTypeTextPlain, fmt.Sprintf("workflow exec status find error %v", nextStatus)) + return + } + } + // 判断是否执行 + isPass, err2 := next.IsPass(ctx) + if err2 != nil { + log.Errorf("check pass find error:%v", err2) + _ = utils.SendResponse(500, "api-workflow.task_check_paas_failed", utils.MimeTypeTextPlain, fmt.Sprintf("check pass find error:%v", err2)) + return + } + if isPass { + log.Debugf("source is %s,target is %s,workflow is pass ", next.Source, next.Target) + nextStatus = ctx.GetContext(WorkflowExecStatus).(map[string]int) + nextStatus[next.Target] = nextStatus[next.Target] - 1 + ctx.SetContext(WorkflowExecStatus, nextStatus) + continue + + } + + // 执行下一步 + err = recursive(next, headers_, responseBody, depth+1, config, log, ctx) + if err != nil { + log.Errorf("recursive error:%v", err) + _ = utils.SendResponse(500, "api-workflow.recursive_failed", utils.MimeTypeTextPlain, fmt.Sprintf("recursive error:%v", err)) + return + } + } + } + + } else { + // statusCode >= 400 ,task httpCall执行失败，放行请求，打印错误，结束workflow + log.Errorf("workflow exec task find error,code is %d,body is %s", statusCode, string(responseBody)) + _ = utils.SendResponse(500, "api-workflow.httpCall_failed", utils.MimeTypeTextPlain, fmt.Sprintf("workflow exec task find error,code is %d,body is %s", statusCode, string(responseBody))) + } + return + + }, config.Env.MaxDepth*config.Env.Timeout) + if err != nil { + log.Errorf("httpcall error:%v", err) + } + + return err +} diff --git a/plugins/wasm-go/extensions/api-workflow/utils/conditional.go b/plugins/wasm-go/extensions/api-workflow/utils/conditional.go new file mode 100644 index 0000000000..03bf0c9954 --- /dev/null +++ b/plugins/wasm-go/extensions/api-workflow/utils/conditional.go @@ -0,0 +1,116 @@ +package utils + +import ( + "fmt" + "regexp" + "strconv" + "strings" +) + +// 原子表达式描述: +// eq arg1 arg2: arg1 == arg2时为true +// ne arg1 arg2: arg1 != arg2时为true +// lt arg1 arg2: arg1 < arg2时为true +// le arg1 arg2: arg1 <= arg2时为true +// gt arg1 arg2: arg1 > arg2时为true +// ge arg1 arg2: arg1 >= arg2时为true +// and arg1 arg2: arg1 && arg2 +// or arg1 arg2: arg1 || arg2 +// contain arg1 arg2: arg1 包含 arg2时为true +var operators = map[string]interface{}{ + "eq": func(a, b interface{}) bool { + return fmt.Sprintf("%v", a) == fmt.Sprintf("%v", b) + }, + "ge": func(a, b float64) bool { return a >= b }, + "le": func(a, b float64) bool { return a <= b }, + "gt": func(a, b float64) bool { return a > b }, + "lt": func(a, b float64) bool { return a < b }, + "and": func(a, b bool) bool { return a && b }, + "or": func(a, b bool) bool { return a || b }, + "contain": func(a, b string) bool { return strings.Contains(a, b) }, +} + +// 执行判断条件 +func ExecConditionalStr(conditionalStr string) (bool, error) { + // 正则表达式匹配括号内的表达式 + re := regexp.MustCompile(`\(([^()]*)\)`) + matches := re.FindAllStringSubmatch(conditionalStr, -1) + // 找到最里面的(原子表达式) + for _, match := range matches { + subCondition := match[1] + result, err := ExecConditionalStr(subCondition) + if err != nil { + return false, err + } + // 用结果替换原子表达式 + conditionalStr = strings.ReplaceAll(conditionalStr, match[0], fmt.Sprintf("%t", result)) + } + + fields := strings.Fields(conditionalStr) + // 执行原子表达式 + if len(fields) == 3 { + compareFunc := operators[fields[0]] + switch fc := compareFunc.(type) { + default: + return false, fmt.Errorf("invalid conditional func %v", compareFunc) + case func(a, b float64) bool: + a, err := strconv.ParseFloat(fields[1], 64) + if err != nil { + return false, fmt.Errorf("invalid conditional str %s", conditionalStr) + } + b, err := strconv.ParseFloat(fields[2], 64) + if err != nil { + return false, fmt.Errorf("invalid conditional str %s", conditionalStr) + } + return fc(a, b), nil + case func(a, b bool) bool: + a, err := strconv.ParseBool(fields[1]) + if err != nil { + return false, fmt.Errorf("invalid conditional str %s", conditionalStr) + } + b, err := strconv.ParseBool(fields[2]) + if err != nil { + return false, fmt.Errorf("invalid conditional str %s", conditionalStr) + } + return fc(a, b), nil + case func(a, b string) bool: + a := fields[1] + b := fields[2] + return fc(a, b), nil + case func(a, b interface{}) bool: + a := fields[1] + b := fields[2] + return fc(a, b), nil + } + // 继续获取上一层的(原子表达式) + } else if strings.Contains(conditionalStr, "(") || strings.Contains(conditionalStr, ")") { + return ExecConditionalStr(conditionalStr) + // 原子表达式有问题，返回 + } else { + return false, fmt.Errorf("invalid conditional str %s", conditionalStr) + } + +} + +// 通过正则表达式寻找模板中的 ｛｛foo｝｝ 字符串foo +// 返回 {{foo}} : foo +func ParseTmplStr(tmpl string) map[string]string { + result := make(map[string]string) + re := regexp.MustCompile(`\{\{(.*?)\}\}`) + matches := re.FindAllStringSubmatch(tmpl, -1) + for _, match := range matches { + result[match[0]] = match[1] + } + return result +} + +// 使用kv替换模板中的字符 +// 例如 模板是`hello,{{foo}}` 使用{"{{foo}}":"bot"} 替换后为`hello,bot` +func ReplacedStr(tmpl string, kvs map[string]string) string { + + for k, v := range kvs { + tmpl = strings.Replace(tmpl, k, v, -1) + } + + return tmpl +} diff --git a/plugins/wasm-go/extensions/api-workflow/utils/conditional_test.go b/plugins/wasm-go/extensions/api-workflow/utils/conditional_test.go new file mode 100644 index 0000000000..b167dc1549 --- /dev/null +++ b/plugins/wasm-go/extensions/api-workflow/utils/conditional_test.go @@ -0,0 +1,100 @@ +package utils + +import ( + "reflect" + "testing" +) + +func TestExecConditionalStr(t *testing.T) { + + tests := []struct { + name string + args string + want bool + wantErr bool + }{ + {"eq int true", "eq 1 1", true, false}, + {"eq int false", "eq 1 2", false, false}, + {"eq str true", "eq foo foo", true, false}, + {"eq str false", "eq foo boo", false, false}, + {"eq float true", "eq 0.99 0.99", true, false}, + {"eq float false", "eq 1.1 2.2", false, false}, + {"eq float int false", "eq 1.0 1", false, false}, + {"eq float str false", "eq 1.0 foo", false, false}, + {"lt true", "lt 1.1 2", true, false}, + {"lt false", "lt 2 1", false, false}, + {"le true", "le 1 2", true, false}, + {"le false", "le 2 1", false, false}, + {"gt true", "gt 2 1", true, false}, + {"gt false", "gt 1 2", false, false}, + {"ge true", "ge 2 1", true, false}, + {"ge false", "ge 1 2", false, false}, + {"and true", "and true true", true, false}, + {"and false", "and true false", false, false}, + {"or true", "or true false", true, false}, + {"or false", "or false false", false, false}, + {"contain true", "contain helloworld world", true, false}, + {"contain false", "contain helloworld moon", false, false}, + {"invalid input", "invalid", false, true}, + {"nested expression 1", "and (eq 1 1) (lt 2 3)", true, false}, + {"nested expression 2", "or (eq 1 2) (and (eq 1 1) (gt 2 3))", false, false}, + {"nested expression error", "or (eq 1 2) (and (eq 1 1) (gt 2 3)))", false, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ExecConditionalStr(tt.args) + if (err != nil) != tt.wantErr { + t.Errorf("ExecConditionalStr() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("ExecConditionalStr() got = %v, want %v", got, tt.want) + } + }) + } +} + +func TestParseTmplStr(t *testing.T) { + type args struct { + tmpl string + } + tests := []struct { + name string + args string + want map[string]string + }{ + {"normal", "{{foo}}", map[string]string{"{{foo}}": "foo"}}, + {"single", "{foo}", map[string]string{}}, + {"empty", "foo", map[string]string{}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := ParseTmplStr(tt.args) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("ParseTmplStr() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestReplacedStr(t *testing.T) { + type args struct { + tmpl string + kvs map[string]string + } + tests := []struct { + name string + args args + want string + }{ + {"normal", args{tmpl: "hello,{{foo}}", kvs: map[string]string{"{{foo}}": "bot"}}, "hello,bot"}, + {"empty", args{tmpl: "hello,foo", kvs: map[string]string{"{{foo}}": "bot"}}, "hello,foo"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ReplacedStr(tt.args.tmpl, tt.args.kvs); got != tt.want { + t.Errorf("ReplacedStr() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/plugins/wasm-go/extensions/api-workflow/utils/http.go b/plugins/wasm-go/extensions/api-workflow/utils/http.go new file mode 100644 index 0000000000..8a9ebb1911 --- /dev/null +++ b/plugins/wasm-go/extensions/api-workflow/utils/http.go @@ -0,0 +1,45 @@ +package utils + +import "github.com/higress-group/proxy-wasm-go-sdk/proxywasm" + +const ( + HeaderContentType = "Content-Type" + + MimeTypeTextPlain = "text/plain" + MimeTypeApplicationJson = "application/json" +) + +func SendResponse(statusCode uint32, statusCodeDetails string, contentType, body string) error { + return proxywasm.SendHttpResponseWithDetail(statusCode, statusCodeDetails, CreateHeaders(HeaderContentType, contentType), []byte(body), -1) +} + +func CreateHeaders(kvs ...string) [][2]string { + headers := make([][2]string, 0, len(kvs)/2) + for i := 0; i < len(kvs); i += 2 { + headers = append(headers, [2]string{kvs[i], kvs[i+1]}) + } + return headers +} + +func OverwriteRequestHost(host string) error { + if originHost, err := proxywasm.GetHttpRequestHeader(":authority"); err == nil { + _ = proxywasm.ReplaceHttpRequestHeader("X-ENVOY-ORIGINAL-HOST", originHost) + } + return proxywasm.ReplaceHttpRequestHeader(":authority", host) +} + +func OverwriteRequestPath(path string) error { + if originPath, err := proxywasm.GetHttpRequestHeader(":path"); err == nil { + _ = proxywasm.ReplaceHttpRequestHeader("X-ENVOY-ORIGINAL-PATH", originPath) + } + return proxywasm.ReplaceHttpRequestHeader(":path", path) +} + +func OverwriteRequestAuthorization(credential string) error { + if exist, _ := proxywasm.GetHttpRequestHeader("X-HI-ORIGINAL-AUTH"); exist == "" { + if originAuth, err := proxywasm.GetHttpRequestHeader("Authorization"); err == nil { + _ = proxywasm.AddHttpRequestHeader("X-HI-ORIGINAL-AUTH", originAuth) + } + } + return proxywasm.ReplaceHttpRequestHeader("Authorization", credential) +} diff --git a/plugins/wasm-go/extensions/api-workflow/utils/tools.go b/plugins/wasm-go/extensions/api-workflow/utils/tools.go new file mode 100644 index 0000000000..2f62e82e9d --- /dev/null +++ b/plugins/wasm-go/extensions/api-workflow/utils/tools.go @@ -0,0 +1,7 @@ +package utils + +import "strings" + +func TrimQuote(source string) string { + return strings.Trim(source, `"`) +} diff --git a/plugins/wasm-go/extensions/api-workflow/workflow/workflow.go b/plugins/wasm-go/extensions/api-workflow/workflow/workflow.go new file mode 100644 index 0000000000..3b827c68ed --- /dev/null +++ b/plugins/wasm-go/extensions/api-workflow/workflow/workflow.go @@ -0,0 +1,325 @@ +package workflow + +import ( + "fmt" + "strings" + + "api-workflow/utils" + + "github.com/alibaba/higress/plugins/wasm-go/pkg/wrapper" + "github.com/tidwall/gjson" + "github.com/tidwall/sjson" +) + +const ( + TaskTypeHTTP string = "http" + TaskStart string = "start" + TaskEnd string = "end" + TaskContinue string = "continue" + UseContextFlag string = "||" + AllFlag string = "@all" +) + +type PluginConfig struct { + + // @Title zh-CN 工作流 + // @Description zh-CN 工作流的具体描述 + Workflow Workflow `json:"workflow" yaml:"workflow"` + // @Title zh-CN 环境变量 + // @Description zh-CN 用来定义整个工作流的环境变量 + Env Env `json:"env" yaml:"env"` +} + +type Env struct { + // @Title zh-CN 超时时间 + // @Description zh-CN 用来定义工作流的超时时间，单位是毫秒 + Timeout uint32 `json:"timeout" yaml:"timeout"` + // @Title zh-CN 最大迭代深度 + // @Description zh-CN 用来定义工作流最大的迭代深度，默认是100 + MaxDepth uint32 `json:"max_depth" yaml:"max_depth"` +} +type Workflow struct { + // @Title zh-CN 边的列表 + // @Description zh-CN 边的列表 + Edges []Edge `json:"edges" yaml:"edges"` + // @Title zh-CN 节点的列表 + // @Description zh-CN 节点的列表 + Nodes map[string]Node `json:"nodes" yaml:"nodes"` + // @Title zh-CN 工作流的状态 + // @Description zh-CN 工作流的执行状态，用于记录node之间的相互依赖和执行情况 + WorkflowExecStatus map[string]int `json:"-" yaml:"-"` +} + +type Edge struct { + // @Title zh-CN 上一步节点 + // @Description zh-CN 上一步节点，必须是定义node的name，或者初始化工作流的start + Source string `json:"source" yaml:"source"` + // @Title zh-CN 当前执行的节点 + // @Description zh-CN 当前执行节点，必须是定义的node的name，或者结束工作流的关键字 end continue + Target string `json:"target" yaml:"target"` + // @Title zh-CN 执行操作 + // @Description zh-CN 执行单元，里面实时封装需要的数据 + Task *Task + // @Title zh-CN 判断表达式 + // @Description zh-CN 是否执行下一步的判断条件 + Conditional string `json:"conditional" yaml:"conditional"` +} + +type Task struct { + Cluster wrapper.Cluster `json:"-" yaml:"-"` + ServicePath string `json:"service_path" yaml:"service_path"` + ServicePort int64 `json:"service_port" yaml:"service_port"` + ServiceKey string `json:"service_key" yaml:"service_key"` + Body []byte `json:"-" yaml:"-"` + Headers [][2]string `json:"headers" yaml:"headers"` + Method string `json:"method" yaml:"method"` + TaskType string `json:"task_type" yaml:"task_type"` +} + +type Node struct { + // @Title zh-CN 节点名称 + // @Description zh-CN 节点名称全局唯一 + Name string `json:"name" yaml:"name"` + // @Title zh-CN 服务名称 + // @Description zh-CN 带服务类型的完整名称，例如 my.dns or foo.static + ServiceName string `json:"service_name" yaml:"service_name"` + // @Title zh-CN 服务端口 + // @Description zh-CN static类型默认是80 + ServicePort int64 `json:"service_port" yaml:"service_port"` + // @Title zh-CN 服务域名 + // @Description zh-CN 服务域名，例如 dashscope.aliyuncs.com + ServiceDomain string `json:"service_domain" yaml:"service_domain"` + // @Title zh-CN http访问路径 + // @Description zh-CN http访问路径，默认是 / + ServicePath string `json:"service_path" yaml:"service_path"` + // @Title zh-CN http 方法 + // @Description zh-CN http方法，支持所有可用方法 GET，POST等 + ServiceMethod string `json:"service_method" yaml:"service_method"` + // @Title zh-CN http 请求头文件 + // @Description zh-CN 请求头文件 + ServiceHeaders []ServiceHeader `json:"service_headers" yaml:"service_headers"` + // @Title zh-CN http 请求body模板 + // @Description zh-CN 请求body模板，用来构造请求 + ServiceBodyTmpl string `json:"service_body_tmpl" yaml:"service_body_tmpl"` + // @Title zh-CN http 请求body模板替换键值对 + // @Description zh-CN 请求body模板替换键值对，用来构造请求。to表示填充的位置，from表示数据从哪里， + // 标识表达式基于 [GJSON PATH](https://github.com/tidwall/gjson/blob/master/SYNTAX.md) 语法提取字符串 + ServiceBodyReplaceKeys []BodyReplaceKeyPair `json:"service_body_replace_keys" yaml:"service_body_replace_keys"` +} +type BodyReplaceKeyPair struct { + // @Title zh-CN from表示数据从哪里， + // @Description zh-CN from表示数据从哪里 + // 标识表达式基于 [GJSON PATH](https://github.com/tidwall/gjson/blob/master/SYNTAX.md) 语法提取字符串 + From string `json:"from" yaml:"from"` + // @Title zh-CN to表示填充的位置 + // @Description zh-CN to表示填充的位置， + // 标识表达式基于 [GJSON PATH](https://github.com/tidwall/gjson/blob/master/SYNTAX.md) 语法提取字符串 + To string `json:"to" yaml:"to"` +} +type ServiceHeader struct { + Key string `json:"key" yaml:"key"` + Value string `json:"value" yaml:"value"` +} + +func (w *Edge) IsEnd() bool { + if w.Target == TaskEnd { + return true + } + return false +} +func (w *Edge) IsContinue() bool { + if w.Target == TaskContinue { + return true + } + return false +} +func (e *Edge) IsPass(ctx wrapper.HttpContext) (bool, error) { + // 执行判断Conditional + if e.Conditional != "" { + + var err error + // 获取模板里的表达式 + + e.Conditional, err = e.WrapperDataByTmplStr(e.Conditional, ctx) + if err != nil { + return false, fmt.Errorf("workflow WrapperDateByTmplStr %s failed: %v", e.Conditional, err) + } + ok, err := e.ExecConditional() + if err != nil { + + return false, fmt.Errorf("wl exec conditional %s failed: %v", e.Conditional, err) + } + return !ok, nil + + } + return false, nil +} + +func (w *Edge) WrapperTask(config PluginConfig, ctx wrapper.HttpContext) error { + + // 判断 node 是否存在 + node, isTool := config.Workflow.Nodes[w.Target] + + if isTool { + w.Task.TaskType = TaskTypeHTTP + } else { + return fmt.Errorf("do not find target :%s", w.Target) + } + + switch w.Task.TaskType { + default: + return fmt.Errorf("unknown node type :%s", w.Task.TaskType) + case TaskTypeHTTP: + err := w.wrapperNodeTask(node, ctx) + if err != nil { + return err + } + + } + return nil + +} + +func (w *Edge) wrapperBody(requestBodyTemplate string, keyPairs []BodyReplaceKeyPair, ctx wrapper.HttpContext) error { + + requestBody, err := w.WrapperDataByTmplStrAndKeys(requestBodyTemplate, keyPairs, ctx) + if err != nil { + return fmt.Errorf("wrapper date by tmpl str is %s ,find err: %v", requestBodyTemplate, err) + } + + w.Task.Body = requestBody + return nil +} + +func (w *Edge) wrapperNodeTask(node Node, ctx wrapper.HttpContext) error { + // 封装cluster + w.Task.Cluster = wrapper.FQDNCluster{ + Host: node.ServiceDomain, + FQDN: node.ServiceName, + Port: node.ServicePort, + } + + // 封装请求body + err := w.wrapperBody(node.ServiceBodyTmpl, node.ServiceBodyReplaceKeys, ctx) + if err != nil { + return fmt.Errorf("wrapper body parse failed: %v", err) + } + + // 封装请求Method path headers + w.Task.Method = node.ServiceMethod + w.Task.ServicePath = node.ServicePath + w.Task.Headers = make([][2]string, 0) + if len(node.ServiceHeaders) > 0 { + for _, header := range node.ServiceHeaders { + w.Task.Headers = append(w.Task.Headers, [2]string{header.Key, header.Value}) + } + } + + return nil +} + +// 利用模板和替换键值对构造请求，使用`||`分隔，str1代表使用node是执行结果。tr2代表如何取数据，使用gjson的表达式，`@all`代表全都要 +func (w *Edge) WrapperDataByTmplStrAndKeys(tmpl string, keyPairs []BodyReplaceKeyPair, ctx wrapper.HttpContext) ([]byte, error) { + var err error + // 不需要替换 node.service_body_replace_keys 为空 + if len(keyPairs) == 0 { + return []byte(tmpl), nil + } + + for _, keyPair := range keyPairs { + + jsonPath := keyPair.From + target := keyPair.To + var contextValueRaw []byte + // 获取上下文数据 + if strings.Contains(jsonPath, UseContextFlag) { + pathStr := strings.Split(jsonPath, UseContextFlag) + if len(pathStr) == 2 { + contextKey := pathStr[0] + contextBody := ctx.GetContext(contextKey) + if contextValue, ok := contextBody.([]byte); ok { + contextValueRaw = contextValue + jsonPath = pathStr[1] + } else { + return nil, fmt.Errorf("context value is not []byte,key is %s", contextKey) + } + } + } + + // 执行封装 ， `@all`代表全都要 + requestBody := gjson.ParseBytes(contextValueRaw) + if jsonPath == AllFlag { + + tmpl, err = sjson.SetRaw(tmpl, target, requestBody.Raw) + if err != nil { + return nil, fmt.Errorf("wrapper body parse failed: %v", err) + } + continue + } + requestBodyJson := requestBody.Get(jsonPath) + if requestBodyJson.Exists() { + tmpl, err = sjson.SetRaw(tmpl, target, requestBodyJson.Raw) + if err != nil { + return nil, fmt.Errorf("wrapper body parse failed: %v", err) + } + + } else { + return nil, fmt.Errorf("wrapper body parse failed: not exists %s", jsonPath) + } + } + return []byte(tmpl), nil + +} + +// 变量使用`{{str1||str2}}`包裹，使用`||`分隔，str1代表使用node是执行结果。tr2代表如何取数据，使用gjson的表达式，`@all`代表全都要 +func (w *Edge) WrapperDataByTmplStr(tmpl string, ctx wrapper.HttpContext) (string, error) { + var body []byte + // 获取模板里的表达式 + TmplKeyAndPath := utils.ParseTmplStr(tmpl) + if len(TmplKeyAndPath) == 0 { + return tmpl, nil + } + // 解析表达式 { "{{str1||str2}}":"str1||str2" } + for k, path := range TmplKeyAndPath { + // 变量使用`{{str1||str2}}`包裹，使用`||`分隔，str1代表使用前面命名为name的数据()。 + if strings.Contains(path, UseContextFlag) { + pathStr := strings.Split(path, UseContextFlag) + if len(pathStr) == 2 { + contextKey := pathStr[0] + contextBody := ctx.GetContext(contextKey) + if contextValue, ok := contextBody.([]byte); ok { + body = contextValue + path = pathStr[1] + } else { + return tmpl, fmt.Errorf("context value is not []byte,key is %s", contextKey) + } + } + // 执行封装 ， `@all`代表全都要 + requestBody := gjson.ParseBytes(body) + if path == AllFlag { + tmpl = strings.Replace(tmpl, k, utils.TrimQuote(requestBody.Raw), -1) + continue + } + requestBodyJson := requestBody.Get(path) + if requestBodyJson.Exists() { + tmpl = utils.ReplacedStr(tmpl, map[string]string{k: utils.TrimQuote(requestBodyJson.Raw)}) + } else { + return tmpl, fmt.Errorf("use path {{%s}} get value is not exists,json is:%s", path, requestBody.Raw) + } + } else { + return "", fmt.Errorf("tmpl parse find error: || is not exists %s", path) + } + + } + return tmpl, nil +} + +func (w *Edge) ExecConditional() (bool, error) { + + ConditionalResult, err := utils.ExecConditionalStr(w.Conditional) + if err != nil { + return false, fmt.Errorf("exec conditional failed: %v", err) + } + return ConditionalResult, nil + +} diff --git a/plugins/wasm-go/extensions/frontend-gray/main.go b/plugins/wasm-go/extensions/frontend-gray/main.go index a6207e7922..81eb3034da 100644 --- a/plugins/wasm-go/extensions/frontend-gray/main.go +++ b/plugins/wasm-go/extensions/frontend-gray/main.go @@ -68,15 +68,16 @@ func onHttpRequestHeaders(ctx wrapper.HttpContext, grayConfig config.GrayConfig, // 如果没有配置比例，则进行灰度规则匹配 if isPageRequest { - log.Infof("grayConfig.TotalGrayWeight==== %v", grayConfig.TotalGrayWeight) if grayConfig.TotalGrayWeight > 0 { + log.Infof("grayConfig.TotalGrayWeight: %v", grayConfig.TotalGrayWeight) deployment = util.FilterGrayWeight(&grayConfig, preVersion, preUniqueClientId, uniqueClientId) } else { deployment = util.FilterGrayRule(&grayConfig, grayKeyValue) } log.Infof("index deployment: %v, path: %v, backend: %v, xPreHigressVersion: %s,%s", deployment, path, deployment.BackendVersion, preVersion, preUniqueClientId) } else { - deployment = util.GetVersion(grayConfig, deployment, preVersion, isPageRequest) + grayDeployment := util.FilterGrayRule(&grayConfig, grayKeyValue) + deployment = util.GetVersion(grayConfig, grayDeployment, preVersion, isPageRequest) } proxywasm.AddHttpRequestHeader(config.XHigressTag, deployment.Version) diff --git a/plugins/wasm-go/extensions/frontend-gray/util/utils.go b/plugins/wasm-go/extensions/frontend-gray/util/utils.go index ab7b2d4752..80291a2c3c 100644 --- a/plugins/wasm-go/extensions/frontend-gray/util/utils.go +++ b/plugins/wasm-go/extensions/frontend-gray/util/utils.go @@ -184,7 +184,7 @@ func GetVersion(grayConfig config.GrayConfig, deployment *config.Deployment, xPr } } } - return grayConfig.BaseDeployment + return deployment } // 从cookie中解析出灰度信息 @@ -294,12 +294,12 @@ func InjectContent(originalHtml string, injectionConfig *config.Injection) strin modifiedHtml := sb.String() - // 注入到头部 - modifiedHtml = strings.ReplaceAll(modifiedHtml, "", headInjection + "\n") - // 注入到body头 - modifiedHtml = strings.ReplaceAll(modifiedHtml, "", "\n" + bodyFirstInjection) - // 注入到body尾 - modifiedHtml = strings.ReplaceAll(modifiedHtml, "", bodyLastInjection + "\n") - - return modifiedHtml + // 注入到头部 + modifiedHtml = strings.ReplaceAll(modifiedHtml, "", headInjection+"\n") + // 注入到body头 + modifiedHtml = strings.ReplaceAll(modifiedHtml, "", "\n"+bodyFirstInjection) + // 注入到body尾 + modifiedHtml = strings.ReplaceAll(modifiedHtml, "", bodyLastInjection+"\n") + + return modifiedHtml } diff --git a/plugins/wasm-rust/Cargo.lock b/plugins/wasm-rust/Cargo.lock index 06dc3d05ea..899d559b0d 100644 --- a/plugins/wasm-rust/Cargo.lock +++ b/plugins/wasm-rust/Cargo.lock @@ -20,12 +20,24 @@ version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" +[[package]] +name = "bytes" +version = "1.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" + [[package]] name = "cfg-if" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "getrandom" version = "0.2.15" @@ -51,6 +63,8 @@ dependencies = [ name = "higress-wasm-rust" version = "0.1.0" dependencies = [ + "http", + "lazy_static", "multimap", "proxy-wasm", "serde", @@ -58,12 +72,29 @@ dependencies = [ "uuid", ] +[[package]] +name = "http" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "itoa" version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + [[package]] name = "libc" version = "0.2.155" diff --git a/plugins/wasm-rust/Cargo.toml b/plugins/wasm-rust/Cargo.toml index ff7ab504ce..a1e5472c63 100644 --- a/plugins/wasm-rust/Cargo.toml +++ b/plugins/wasm-rust/Cargo.toml @@ -11,3 +11,5 @@ serde = "1.0" serde_json = "1.0" uuid = { version = "1.3.3", features = ["v4"] } multimap = "0" +http = "1" +lazy_static = "1" diff --git a/plugins/wasm-rust/extensions/demo-wasm/Cargo.lock b/plugins/wasm-rust/extensions/demo-wasm/Cargo.lock new file mode 100644 index 0000000000..85e2edaea3 --- /dev/null +++ b/plugins/wasm-rust/extensions/demo-wasm/Cargo.lock @@ -0,0 +1,263 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "ahash" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "allocator-api2" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" + +[[package]] +name = "bytes" +version = "1.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "demo-wasm" +version = "0.1.0" +dependencies = [ + "higress-wasm-rust", + "http", + "multimap", + "proxy-wasm", + "serde", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", + "allocator-api2", +] + +[[package]] +name = "higress-wasm-rust" +version = "0.1.0" +dependencies = [ + "http", + "lazy_static", + "multimap", + "proxy-wasm", + "serde", + "serde_json", + "uuid", +] + +[[package]] +name = "http" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "itoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.157" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "374af5f94e54fa97cf75e945cce8a6b201e88a1a07e688b47dfd2a59c66dbd86" + +[[package]] +name = "log" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "multimap" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03" +dependencies = [ + "serde", +] + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "proc-macro2" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "proxy-wasm" +version = "0.2.2" +source = "git+https://github.com/higress-group/proxy-wasm-rust-sdk?branch=main#73833051f57d483570cf5aaa9d62bd7402fae63b" +dependencies = [ + "hashbrown", + "log", +] + +[[package]] +name = "quote" +version = "1.0.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + +[[package]] +name = "serde" +version = "1.0.208" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cff085d2cb684faa248efb494c39b68e522822ac0de72ccf08109abde717cfb2" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.208" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24008e81ff7613ed8e5ba0cfaf24e2c2f1e5b8a0495711e44fcd4882fca62bcf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83c8e735a073ccf5be70aa8066aa984eaf2fa000db6c8d0100ae605b366d31ed" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "syn" +version = "2.0.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6af063034fc1935ede7be0122941bafa9bacb949334d090b77ca98b5817c7d9" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "uuid" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" +dependencies = [ + "getrandom", +] + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/plugins/wasm-rust/extensions/demo-wasm/Cargo.toml b/plugins/wasm-rust/extensions/demo-wasm/Cargo.toml new file mode 100644 index 0000000000..a517c2b531 --- /dev/null +++ b/plugins/wasm-rust/extensions/demo-wasm/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "demo-wasm" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[lib] +crate-type = ["cdylib"] + +[dependencies] +higress-wasm-rust = { path = "../../", version = "0.1.0" } +proxy-wasm = { git="https://github.com/higress-group/proxy-wasm-rust-sdk", branch="main", version="0.2.2" } +serde = { version = "1.0", features = ["derive"] } +multimap = "*" +http = "*" \ No newline at end of file diff --git a/plugins/wasm-rust/extensions/demo-wasm/src/lib.rs b/plugins/wasm-rust/extensions/demo-wasm/src/lib.rs new file mode 100644 index 0000000000..55647a83cc --- /dev/null +++ b/plugins/wasm-rust/extensions/demo-wasm/src/lib.rs @@ -0,0 +1,203 @@ +use higress_wasm_rust::cluster_wrapper::DnsCluster; +use higress_wasm_rust::log::Log; +use higress_wasm_rust::plugin_wrapper::{ + HttpCallArgStorage, HttpCallbackFn, HttpContextWrapper, RootContextWrapper, +}; +use higress_wasm_rust::rule_matcher::{on_configure, RuleMatcher, SharedRuleMatcher}; +use http::Method; +use multimap::MultiMap; +use proxy_wasm::traits::{Context, HttpContext, RootContext}; +use proxy_wasm::types::{Bytes, ContextType, DataAction, HeaderAction, LogLevel}; + +use serde::Deserialize; +use std::cell::RefCell; +use std::ops::DerefMut; +use std::rc::Rc; +use std::time::Duration; + +proxy_wasm::main! {{ + proxy_wasm::set_log_level(LogLevel::Trace); + proxy_wasm::set_root_context(|_|Box::new(DemoWasmRoot::new())); +}} + +const PLUGIN_NAME: &str = "demo-wasm"; + +#[derive(Default, Debug, Deserialize, Clone)] +struct DemoWasmConfig { + // 配置文件结构体 + test: String, +} + +fn format_body(body: Option>) -> String { + if let Some(bd) = &body { + if let Ok(b) = std::str::from_utf8(bd) { + return b.to_string(); + } + } + format!("{:?}", body) +} + +fn test_callback( + this: &mut DemoWasm, + status_code: u16, + headers: &MultiMap, + body: Option>, +) { + this.log.info(&format!( + "test_callback status_code:{}, headers: {:?}, body: {}", + status_code, + headers, + format_body(body) + )); + this.reset_http_request(); +} +struct DemoWasm { + // 每个请求对应的插件实例 + log: Log, + config: Option, + + arg_storage: HttpCallArgStorage>>, +} + +impl Context for DemoWasm {} +impl HttpContext for DemoWasm {} +impl HttpContextWrapper>> for DemoWasm { + fn log(&self) -> &Log { + &self.log + } + fn get_http_call_storage( + &mut self, + ) -> Option<&mut HttpCallArgStorage>>> { + Some(&mut self.arg_storage) + } + fn on_config(&mut self, config: &DemoWasmConfig) { + // 获取config + self.log.info(&format!("on_config {}", config.test)); + self.config = Some(config.clone()) + } + fn on_http_request_complete_headers( + &mut self, + headers: &MultiMap, + ) -> HeaderAction { + // 请求header获取完成回调 + self.log + .info(&format!("on_http_request_complete_headers {:?}", headers)); + HeaderAction::Continue + } + fn on_http_response_complete_headers( + &mut self, + headers: &MultiMap, + ) -> HeaderAction { + // 返回header获取完成回调 + self.log + .info(&format!("on_http_response_complete_headers {:?}", headers)); + HeaderAction::Continue + } + fn cache_request_body(&self) -> bool { + // 是否缓存请求body + true + } + fn cache_response_body(&self) -> bool { + // 是否缓存返回body + true + } + fn on_http_call_response_detail( + &mut self, + _token_id: u32, + arg: Box>, + status_code: u16, + headers: &MultiMap, + body: Option>, + ) { + arg(self, status_code, headers, body) + } + fn on_http_request_complete_body(&mut self, req_body: &Bytes) -> DataAction { + // 请求body获取完成回调 + self.log.info(&format!( + "on_http_request_complete_body {}", + String::from_utf8(req_body.clone()).unwrap_or("".to_string()) + )); + let cluster = DnsCluster::new("httpbin", "httpbin.org", 80); + if self + .http_call( + &cluster, + &Method::POST, + "http://httpbin.org/post", + MultiMap::new(), + Some("test_body".as_bytes()), + // Box::new(move |this, _status_code, _headers, _body| this.resume_http_request()), + Box::new(test_callback), + Duration::from_secs(5), + ) + .is_ok() + { + DataAction::StopIterationAndBuffer + } else { + self.log.info("http_call fail"); + DataAction::Continue + } + } + fn on_http_response_complete_body(&mut self, res_body: &Bytes) -> DataAction { + // 返回body获取完成回调 + self.log.info(&format!( + "on_http_response_complete_body {}", + String::from_utf8(res_body.clone()).unwrap_or("".to_string()) + )); + DataAction::Continue + } +} +struct DemoWasmRoot { + log: Log, + rule_matcher: SharedRuleMatcher, +} +impl DemoWasmRoot { + fn new() -> Self { + let log = Log::new(PLUGIN_NAME.to_string()); + log.info("DemoWasmRoot::new"); + DemoWasmRoot { + log, + rule_matcher: Rc::new(RefCell::new(RuleMatcher::default())), + } + } +} + +impl Context for DemoWasmRoot {} + +impl RootContext for DemoWasmRoot { + fn on_configure(&mut self, _plugin_configuration_size: usize) -> bool { + self.log.info("DemoWasmRoot::on_configure"); + on_configure( + self, + _plugin_configuration_size, + self.rule_matcher.borrow_mut().deref_mut(), + &self.log, + ) + } + fn create_http_context(&self, context_id: u32) -> Option> { + self.log.info(&format!( + "DemoWasmRoot::create_http_context({})", + context_id + )); + self.create_http_context_use_wrapper(context_id) + } + fn get_type(&self) -> Option { + Some(ContextType::HttpContext) + } +} + +impl RootContextWrapper>> for DemoWasmRoot { + fn rule_matcher(&self) -> &SharedRuleMatcher { + &self.rule_matcher + } + + fn create_http_context_wrapper( + &self, + _context_id: u32, + ) -> Option>>>> { + Some(Box::new(DemoWasm { + config: None, + log: Log::new(PLUGIN_NAME.to_string()), + arg_storage: HttpCallArgStorage::new(), + })) + } +} diff --git a/plugins/wasm-rust/src/cluster_wrapper.rs b/plugins/wasm-rust/src/cluster_wrapper.rs new file mode 100644 index 0000000000..2891293fb3 --- /dev/null +++ b/plugins/wasm-rust/src/cluster_wrapper.rs @@ -0,0 +1,259 @@ +use crate::{internal::get_property, request_wrapper::get_request_host}; + +pub trait Cluster { + fn cluster_name(&self) -> String; + fn host_name(&self) -> String; +} +#[derive(Debug, Clone)] +pub struct RouteCluster { + host: String, +} +impl RouteCluster { + pub fn new(host: &str) -> Self { + RouteCluster { + host: host.to_string(), + } + } +} +impl Cluster for RouteCluster { + fn cluster_name(&self) -> String { + if let Some(res) = get_property(vec!["cluster_name"]) { + if let Ok(r) = String::from_utf8(res) { + return r; + } + } + String::new() + } + + fn host_name(&self) -> String { + if !self.host.is_empty() { + return self.host.clone(); + } + + get_request_host() + } +} + +#[derive(Debug, Clone)] +pub struct K8sCluster { + service_name: String, + namespace: String, + port: String, + version: String, + host: String, +} + +impl K8sCluster { + pub fn new(service_name: &str, namespace: &str, port: &str, version: &str, host: &str) -> Self { + K8sCluster { + service_name: service_name.to_string(), + namespace: namespace.to_string(), + port: port.to_string(), + version: version.to_string(), + host: host.to_string(), + } + } +} + +impl Cluster for K8sCluster { + fn cluster_name(&self) -> String { + format!( + "outbound|{}|{}|{}.{}.svc.cluster.local", + self.port, + self.version, + self.service_name, + if self.namespace.is_empty() { + "default" + } else { + &self.namespace + } + ) + } + + fn host_name(&self) -> String { + if self.host.is_empty() { + format!("{}.{}.svc.cluster.local", self.service_name, self.namespace) + } else { + self.host.clone() + } + } +} + +#[derive(Debug, Clone)] +pub struct NacosCluster { + service_name: String, + group: String, + namespace_id: String, + port: u16, + is_ext_registry: bool, + version: String, + host: String, +} + +impl NacosCluster { + pub fn new( + service_name: &str, + group: &str, + namespace_id: &str, + port: u16, + is_ext_registry: bool, + version: &str, + host: &str, + ) -> Self { + NacosCluster { + service_name: service_name.to_string(), + group: group.to_string(), + namespace_id: namespace_id.to_string(), + port, + is_ext_registry, + version: version.to_string(), + host: host.to_string(), + } + } +} +impl Cluster for NacosCluster { + fn cluster_name(&self) -> String { + let group = if self.group.is_empty() { + "DEFAULT-GROUP".to_string() + } else { + self.group.replace('_', "-") + }; + let tail = if self.is_ext_registry { + "nacos-ext" + } else { + "nacos" + }; + format!( + "outbound|{}|{}|{}.{}.{}.{}", + self.port, self.version, self.service_name, group, self.namespace_id, tail + ) + } + + fn host_name(&self) -> String { + if self.host.is_empty() { + self.service_name.clone() + } else { + self.host.clone() + } + } +} + +#[derive(Debug, Clone)] +pub struct StaticIpCluster { + service_name: String, + port: u16, + host: String, +} + +impl StaticIpCluster { + pub fn new(service_name: &str, port: u16, host: &str) -> Self { + StaticIpCluster { + service_name: service_name.to_string(), + port, + host: host.to_string(), + } + } +} +impl Cluster for StaticIpCluster { + fn cluster_name(&self) -> String { + format!("outbound|{}||{}.static", self.port, self.service_name) + } + + fn host_name(&self) -> String { + if self.host.is_empty() { + self.service_name.clone() + } else { + self.host.clone() + } + } +} + +#[derive(Debug, Clone)] +pub struct DnsCluster { + service_name: String, + domain: String, + port: u16, +} + +impl DnsCluster { + pub fn new(service_name: &str, domain: &str, port: u16) -> Self { + DnsCluster { + service_name: service_name.to_string(), + domain: domain.to_string(), + port, + } + } +} +impl Cluster for DnsCluster { + fn cluster_name(&self) -> String { + format!("outbound|{}||{}.dns", self.port, self.service_name) + } + + fn host_name(&self) -> String { + self.domain.clone() + } +} + +#[derive(Debug, Clone)] +pub struct ConsulCluster { + service_name: String, + datacenter: String, + port: u16, + host: String, +} + +impl ConsulCluster { + pub fn new(service_name: &str, datacenter: &str, port: u16, host: &str) -> Self { + ConsulCluster { + service_name: service_name.to_string(), + datacenter: datacenter.to_string(), + port, + host: host.to_string(), + } + } +} +impl Cluster for ConsulCluster { + fn cluster_name(&self) -> String { + format!( + "outbound|{}||{}.{}.consul", + self.port, self.service_name, self.datacenter + ) + } + + fn host_name(&self) -> String { + if self.host.is_empty() { + self.service_name.clone() + } else { + self.host.clone() + } + } +} + +#[derive(Debug, Clone)] +pub struct FQDNCluster { + fqdn: String, + host: String, + port: u16, +} + +impl FQDNCluster { + pub fn new(fqdn: &str, host: &str, port: u16) -> Self { + FQDNCluster { + fqdn: fqdn.to_string(), + host: host.to_string(), + port, + } + } +} +impl Cluster for FQDNCluster { + fn cluster_name(&self) -> String { + format!("outbound|{}||{}", self.port, self.fqdn) + } + fn host_name(&self) -> String { + if self.host.is_empty() { + self.fqdn.clone() + } else { + self.host.clone() + } + } +} diff --git a/plugins/wasm-rust/src/lib.rs b/plugins/wasm-rust/src/lib.rs index 1e38d0cb0b..3296ff648a 100644 --- a/plugins/wasm-rust/src/lib.rs +++ b/plugins/wasm-rust/src/lib.rs @@ -12,8 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +pub mod cluster_wrapper; pub mod error; mod internal; pub mod log; pub mod plugin_wrapper; +pub mod request_wrapper; pub mod rule_matcher; diff --git a/plugins/wasm-rust/src/plugin_wrapper.rs b/plugins/wasm-rust/src/plugin_wrapper.rs index ba4e0a2580..25d445f22f 100644 --- a/plugins/wasm-rust/src/plugin_wrapper.rs +++ b/plugins/wasm-rust/src/plugin_wrapper.rs @@ -12,15 +12,24 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::collections::HashMap; +use std::time::Duration; + +use crate::cluster_wrapper::Cluster; +use crate::log::Log; use crate::rule_matcher::SharedRuleMatcher; +use http::{method::Method, Uri}; +use lazy_static::lazy_static; use multimap::MultiMap; -use proxy_wasm::hostcalls::log; use proxy_wasm::traits::{Context, HttpContext, RootContext}; -use proxy_wasm::types::LogLevel; -use proxy_wasm::types::{Action, Bytes, DataAction, HeaderAction}; +use proxy_wasm::types::{Action, Bytes, DataAction, HeaderAction, Status}; use serde::de::DeserializeOwned; -pub trait RootContextWrapper: RootContext +lazy_static! { + static ref LOG: Log = Log::new("plugin_wrapper".to_string()); +} + +pub trait RootContextWrapper: RootContext where PluginConfig: Default + DeserializeOwned + 'static + Clone, { @@ -39,11 +48,37 @@ where fn create_http_context_wrapper( &self, _context_id: u32, - ) -> Option>> { + ) -> Option>> { None } } -pub trait HttpContextWrapper: HttpContext { +pub type HttpCallbackFn = dyn FnOnce(&mut T, u16, &MultiMap, Option>); + +pub struct HttpCallArgStorage { + args: HashMap, +} +impl Default for HttpCallArgStorage { + fn default() -> Self { + Self::new() + } +} +impl HttpCallArgStorage { + pub fn new() -> Self { + HttpCallArgStorage { + args: HashMap::new(), + } + } + pub fn set(&mut self, token_id: u32, arg: HttpCallArg) { + self.args.insert(token_id, arg); + } + pub fn pop(&mut self, token_id: u32) -> Option { + self.args.remove(&token_id) + } +} +pub trait HttpContextWrapper: HttpContext { + fn log(&self) -> &Log { + &LOG + } fn on_config(&mut self, _config: &PluginConfig) {} fn on_http_request_complete_headers( &mut self, @@ -69,26 +104,96 @@ pub trait HttpContextWrapper: HttpContext { fn on_http_response_complete_body(&mut self, _res_body: &Bytes) -> DataAction { DataAction::Continue } + + #[allow(clippy::too_many_arguments)] + fn on_http_call_response_detail( + &mut self, + _token_id: u32, + _arg: HttpCallArg, + _status_code: u16, + _headers: &MultiMap, + _body: Option>, + ) { + } fn replace_http_request_body(&mut self, body: &[u8]) { self.set_http_request_body(0, i32::MAX as usize, body) } fn replace_http_response_body(&mut self, body: &[u8]) { self.set_http_response_body(0, i32::MAX as usize, body) } + + fn get_http_call_storage(&mut self) -> Option<&mut HttpCallArgStorage> { + None + } + + #[allow(clippy::too_many_arguments)] + fn http_call( + &mut self, + cluster: &dyn Cluster, + method: &Method, + raw_url: &str, + headers: MultiMap, + body: Option<&[u8]>, + arg: HttpCallArg, + timeout: Duration, + ) -> Result { + if let Ok(uri) = raw_url.parse::() { + let mut authority = cluster.host_name(); + if let Some(host) = uri.host() { + authority = host.to_string(); + } + let mut path = uri.path().to_string(); + if let Some(query) = uri.query() { + path = format!("{}?{}", path, query); + } + let mut headers_vec = Vec::new(); + for (k, v) in headers.iter() { + headers_vec.push((k.as_str(), v.as_str())); + } + headers_vec.push((":method", method.as_str())); + headers_vec.push((":path", &path)); + headers_vec.push((":authority", &authority)); + let ret = self.dispatch_http_call( + &cluster.cluster_name(), + headers_vec, + body, + Vec::new(), + timeout, + ); + + if let Ok(token_id) = ret { + if let Some(storage) = self.get_http_call_storage() { + storage.set(token_id, arg); + self.log().debug( + &format!( + "http call start, id: {}, cluster: {}, method: {}, url: {}, body: {:?}, timeout: {:?}", + token_id, cluster.cluster_name(), method.as_str(), raw_url, body, timeout + ) + ); + } else { + return Err(Status::InternalFailure); + } + } + ret + } else { + self.log().critical(&format!("invalid raw_url:{}", raw_url)); + Err(Status::ParseFailure) + } + } } -pub struct PluginHttpWrapper { +pub struct PluginHttpWrapper { req_headers: MultiMap, res_headers: MultiMap, req_body_len: usize, res_body_len: usize, config: Option, rule_matcher: SharedRuleMatcher, - http_content: Box>, + http_content: Box>, } -impl PluginHttpWrapper { +impl PluginHttpWrapper { pub fn new( rule_matcher: &SharedRuleMatcher, - http_content: Box>, + http_content: Box>, ) -> Self { PluginHttpWrapper { req_headers: MultiMap::new(), @@ -100,8 +205,15 @@ impl PluginHttpWrapper { http_content, } } + fn get_http_call_arg(&mut self, token_id: u32) -> Option { + if let Some(storage) = self.http_content.get_http_call_storage() { + storage.pop(token_id) + } else { + None + } + } } -impl Context for PluginHttpWrapper { +impl Context for PluginHttpWrapper { fn on_http_call_response( &mut self, token_id: u32, @@ -109,8 +221,50 @@ impl Context for PluginHttpWrapper { body_size: usize, num_trailers: usize, ) { - self.http_content - .on_http_call_response(token_id, num_headers, body_size, num_trailers) + if let Some(arg) = self.get_http_call_arg(token_id) { + let body = self.get_http_call_response_body(0, body_size); + let mut headers = MultiMap::new(); + let mut status_code = 502; + let mut normal_response = false; + for (k, v) in self.get_http_call_response_headers_bytes() { + match String::from_utf8(v) { + Ok(header_value) => { + if k == ":status" { + if let Ok(code) = header_value.parse::() { + status_code = code; + normal_response = true; + } else { + self.http_content + .log() + .error(&format!("failed to parse status: {}", header_value)); + status_code = 500; + } + } + headers.insert(k, header_value); + } + Err(_) => { + self.http_content.log().warn(&format!( + "http call response header contains non-ASCII characters header: {}", + k + )); + } + } + } + self.http_content.log().warn(&format!( + "http call end, id: {}, code: {}, normal: {}, body: {:?}", + token_id, status_code, normal_response, body + )); + self.http_content.on_http_call_response_detail( + token_id, + arg, + status_code, + &headers, + body, + ) + } else { + self.http_content + .on_http_call_response(token_id, num_headers, body_size, num_trailers) + } } fn on_grpc_call_response(&mut self, token_id: u32, status_code: u32, response_size: usize) { @@ -138,7 +292,7 @@ impl Context for PluginHttpWrapper { self.http_content.on_done() } } -impl HttpContext for PluginHttpWrapper +impl HttpContext for PluginHttpWrapper where PluginConfig: Default + DeserializeOwned + Clone, { @@ -152,15 +306,10 @@ where self.req_headers.insert(k, header_value); } Err(_) => { - log( - LogLevel::Warn, - format!( - "request http header contains non-ASCII characters header: {}", - k - ) - .as_str(), - ) - .unwrap(); + self.http_content.log().warn(&format!( + "request http header contains non-ASCII characters header: {}", + k + )); } } } @@ -212,15 +361,10 @@ where self.res_headers.insert(k, header_value); } Err(_) => { - log( - LogLevel::Warn, - format!( - "response http header contains non-ASCII characters header: {}", - k - ) - .as_str(), - ) - .unwrap(); + self.http_content.log().warn(&format!( + "response http header contains non-ASCII characters header: {}", + k + )); } } } diff --git a/plugins/wasm-rust/src/request_wrapper.rs b/plugins/wasm-rust/src/request_wrapper.rs new file mode 100644 index 0000000000..bc9624f6a9 --- /dev/null +++ b/plugins/wasm-rust/src/request_wrapper.rs @@ -0,0 +1,82 @@ +use proxy_wasm::hostcalls; + +use crate::internal; + +fn get_request_head(head: &str, log_flag: &str) -> String { + if let Some(value) = internal::get_http_request_header(head) { + value + } else { + hostcalls::log( + proxy_wasm::types::LogLevel::Error, + &format!("get request {} failed", log_flag), + ) + .unwrap(); + String::new() + } +} +pub fn get_request_scheme() -> String { + get_request_head(":scheme", "head") +} + +pub fn get_request_host() -> String { + get_request_head(":authority", "host") +} + +pub fn get_request_path() -> String { + get_request_head(":path", "path") +} + +pub fn get_request_method() -> String { + get_request_head(":method", "method") +} + +pub fn is_binary_request_body() -> bool { + if let Some(content_type) = internal::get_http_request_header("content-type") { + if content_type.contains("octet-stream") || content_type.contains("grpc") { + return true; + } + } + if let Some(encoding) = internal::get_http_request_header("content-encoding") { + if !encoding.is_empty() { + return true; + } + } + false +} + +pub fn is_binary_response_body() -> bool { + if let Some(content_type) = internal::get_http_response_header("content-type") { + if content_type.contains("octet-stream") || content_type.contains("grpc") { + return true; + } + } + if let Some(encoding) = internal::get_http_response_header("content-encoding") { + if !encoding.is_empty() { + return true; + } + } + false +} +pub fn has_request_body() -> bool { + let content_type = internal::get_http_request_header("content-type"); + let content_length_str = internal::get_http_request_header("content-length"); + let transfer_encoding = internal::get_http_request_header("transfer-encoding"); + hostcalls::log( + proxy_wasm::types::LogLevel::Debug, + &format!( + "check has request body: content_type:{:?}, content_length_str:{:?}, transfer_encoding:{:?}", + content_type, content_length_str, transfer_encoding + ) + ).unwrap(); + if !content_type.is_some_and(|x| !x.is_empty()) { + return true; + } + if let Some(cl) = content_length_str { + if let Ok(content_length) = cl.parse::() { + if content_length > 0 { + return true; + } + } + } + transfer_encoding.is_some_and(|x| x == "chunked") +} diff --git a/registry/consul/watcher.go b/registry/consul/watcher.go index f88cb88d52..157d278329 100644 --- a/registry/consul/watcher.go +++ b/registry/consul/watcher.go @@ -237,7 +237,7 @@ func (w *watcher) Stop() { // clean the cache suffix := strings.Join([]string{serviceName, w.ConsulDatacenter, w.Type}, common.DotSeparator) host := strings.ReplaceAll(suffix, common.Underscore, common.Hyphen) - w.cache.DeleteServiceEntryWrapper(host) + w.cache.DeleteServiceWrapper(host) } w.isStop = true close(w.stop) @@ -295,15 +295,16 @@ func (w *watcher) getSubscribeCallback(serviceName string) func(idx uint64, data serviceEntry := w.generateServiceEntry(host, services) if serviceEntry != nil { log.Infof("consul update serviceEntry %s cache", host) - w.cache.UpdateServiceEntryWrapper(host, &memory.ServiceEntryWrapper{ + w.cache.UpdateServiceWrapper(host, &memory.ServiceWrapper{ ServiceEntry: serviceEntry, ServiceName: serviceName, Suffix: suffix, RegistryType: w.Type, + RegistryName: w.Name, }) } else { log.Infof("consul serviceEntry %s is nil", host) - //w.cache.DeleteServiceEntryWrapper(host) + //w.cache.DeleteServiceWrapper(host) } } } diff --git a/registry/direct/watcher.go b/registry/direct/watcher.go index 6bc22597ac..f523f1f06a 100644 --- a/registry/direct/watcher.go +++ b/registry/direct/watcher.go @@ -22,14 +22,15 @@ import ( "sync" "istio.io/api/networking/v1alpha3" - "istio.io/istio/pkg/config/protocol" "istio.io/pkg/log" apiv1 "github.com/alibaba/higress/api/networking/v1" "github.com/alibaba/higress/pkg/common" + ingress "github.com/alibaba/higress/pkg/ingress/kube/common" "github.com/alibaba/higress/registry" provider "github.com/alibaba/higress/registry" "github.com/alibaba/higress/registry/memory" + "github.com/go-errors/errors" ) type watcher struct { @@ -48,6 +49,9 @@ func NewWatcher(cache memory.Cache, opts ...WatcherOption) (provider.Watcher, er for _, opt := range opts { opt(w) } + if common.ParseProtocol(w.Protocol) == common.Unsupported { + return nil, errors.Errorf("invalid protocol:%s", w.Protocol) + } return w, nil } @@ -75,17 +79,42 @@ func WithPort(port uint32) WatcherOption { } } +func WithProtocol(protocol string) WatcherOption { + return func(w *watcher) { + w.Protocol = protocol + if w.Protocol == "" { + w.Protocol = string(common.HTTP) + } + } +} + +func WithSNI(sni string) WatcherOption { + return func(w *watcher) { + w.Sni = sni + } +} + func (w *watcher) Run() { w.mutex.Lock() defer w.mutex.Unlock() host := strings.Join([]string{w.Name, w.Type}, common.DotSeparator) serviceEntry := w.generateServiceEntry(host) if serviceEntry != nil { - w.cache.UpdateServiceEntryWrapper(host, &memory.ServiceEntryWrapper{ - ServiceName: w.Name, - ServiceEntry: serviceEntry, - Suffix: w.Type, - RegistryType: w.Type, + var destinationRuleWrapper *ingress.WrapperDestinationRule + destinationRule := w.generateDestinationRule(serviceEntry) + if destinationRule != nil { + destinationRuleWrapper = &ingress.WrapperDestinationRule{ + DestinationRule: destinationRule, + ServiceKey: ingress.CreateMcpServiceKey(host, int32(w.Port)), + } + } + w.cache.UpdateServiceWrapper(host, &memory.ServiceWrapper{ + ServiceName: w.Name, + ServiceEntry: serviceEntry, + Suffix: w.Type, + RegistryType: w.Type, + RegistryName: w.Name, + DestinationRuleWrapper: destinationRuleWrapper, }) w.UpdateService() } @@ -96,7 +125,7 @@ func (w *watcher) Stop() { w.mutex.Lock() defer w.mutex.Unlock() host := strings.Join([]string{w.Name, w.Type}, common.DotSeparator) - w.cache.DeleteServiceEntryWrapper(host) + w.cache.DeleteServiceWrapper(host) w.Ready(false) } @@ -146,8 +175,8 @@ func (w *watcher) generateServiceEntry(host string) *v1alpha3.ServiceEntry { var ports []*v1alpha3.ServicePort ports = append(ports, &v1alpha3.ServicePort{ Number: w.Port, - Name: "http", - Protocol: string(protocol.HTTP), + Name: w.Protocol, + Protocol: string(common.ParseProtocol(w.Protocol)), }) se := &v1alpha3.ServiceEntry{ Hosts: []string{host}, @@ -163,6 +192,34 @@ func (w *watcher) generateServiceEntry(host string) *v1alpha3.ServiceEntry { return se } +func (w *watcher) generateDestinationRule(se *v1alpha3.ServiceEntry) *v1alpha3.DestinationRule { + if !common.Protocol(se.Ports[0].Protocol).IsHTTPS() { + return nil + } + sni := w.Sni + // DNS type, automatically sets SNI based on domain name. + if sni == "" && w.Type == string(registry.DNS) && len(se.Endpoints) == 1 { + sni = w.Domain + } + return &v1alpha3.DestinationRule{ + Host: se.Hosts[0], + TrafficPolicy: &v1alpha3.TrafficPolicy{ + PortLevelSettings: []*v1alpha3.TrafficPolicy_PortTrafficPolicy{ + &v1alpha3.TrafficPolicy_PortTrafficPolicy{ + Port: &v1alpha3.PortSelector{ + Number: se.Ports[0].Number, + }, + Tls: &v1alpha3.ClientTLSSettings{ + Mode: v1alpha3.ClientTLSSettings_SIMPLE, + Sni: sni, + }, + }, + }, + }, + } + +} + func (w *watcher) GetRegistryType() string { return w.RegistryConfig.Type } diff --git a/registry/eureka/client/http_client.go b/registry/eureka/client/http_client.go index 8203130073..bfec44cfe6 100644 --- a/registry/eureka/client/http_client.go +++ b/registry/eureka/client/http_client.go @@ -18,6 +18,7 @@ import ( "encoding/json" "fmt" "io" + "net" "net/http" "time" @@ -125,13 +126,25 @@ func (c *eurekaHttpClient) getApplications(path string) (*Applications, error) { apps := map[string]*fargo.Application{} for idx := range rj.Response.Applications { + ignore := false app := rj.Response.Applications[idx] + for _, instance := range app.Instances { + if ip := net.ParseIP(instance.IPAddr); ip == nil { + log.Warnf("the Non-IP IPAddr %s is not allowed, please check your app: %s", instance.IPAddr, app.Name) + ignore = true + break + } + } + if ignore { + continue + } apps[app.Name] = app } for name, app := range apps { log.Debugf("Parsing metadata for app %v", name) if err := app.ParseAllMetadata(); err != nil { + log.Errorf("Failed to parse metadata for app %v: %v", name, err) return nil, err } } diff --git a/registry/eureka/watcher.go b/registry/eureka/watcher.go index 7c6d5c27ca..280c4c27e3 100644 --- a/registry/eureka/watcher.go +++ b/registry/eureka/watcher.go @@ -147,7 +147,7 @@ func (w *watcher) Stop() { log.Errorf("Failed to unsubscribe service : %v", serviceName) continue } - w.cache.DeleteServiceEntryWrapper(makeHost(serviceName)) + w.cache.DeleteServiceWrapper(makeHost(serviceName)) } w.UpdateService() } @@ -203,17 +203,18 @@ func (w *watcher) subscribe(service *fargo.Application) error { if err != nil { return err } - w.cache.UpdateServiceEntryWrapper(makeHost(service.Name), &memory.ServiceEntryWrapper{ + w.cache.UpdateServiceWrapper(makeHost(service.Name), &memory.ServiceWrapper{ ServiceName: service.Name, ServiceEntry: se, Suffix: suffix, RegistryType: w.Type, + RegistryName: w.Name, }) return nil } if w.updateCacheWhenEmpty { - w.cache.DeleteServiceEntryWrapper(makeHost(service.Name)) + w.cache.DeleteServiceWrapper(makeHost(service.Name)) } return nil diff --git a/registry/memory/cache.go b/registry/memory/cache.go index c0cacf7497..9e72a99778 100644 --- a/registry/memory/cache.go +++ b/registry/memory/cache.go @@ -24,26 +24,28 @@ import ( "istio.io/pkg/log" "github.com/alibaba/higress/pkg/common" + ingress "github.com/alibaba/higress/pkg/ingress/kube/common" ) type Cache interface { - UpdateServiceEntryWrapper(service string, data *ServiceEntryWrapper) - DeleteServiceEntryWrapper(service string) + UpdateServiceWrapper(service string, data *ServiceWrapper) + DeleteServiceWrapper(service string) PurgeStaleService() UpdateServiceEntryEndpointWrapper(service, ip, regionId, zoneId, protocol string, labels map[string]string) GetServiceByEndpoints(requestVersions, endpoints map[string]bool, versionKey string, protocol common.Protocol) map[string][]string GetAllServiceEntry() []*v1alpha3.ServiceEntry - GetAllServiceEntryWrapper() []*ServiceEntryWrapper - GetIncrementalServiceEntryWrapper() (updatedList []*ServiceEntryWrapper, deletedList []*ServiceEntryWrapper) + GetAllServiceWrapper() []*ServiceWrapper + GetAllDestinationRuleWrapper() []*ingress.WrapperDestinationRule + GetIncrementalServiceWrapper() (updatedList []*ServiceWrapper, deletedList []*ServiceWrapper) RemoveEndpointByIp(ip string) } func NewCache() Cache { return &store{ mux: &sync.RWMutex{}, - sew: make(map[string]*ServiceEntryWrapper), - toBeUpdated: make([]*ServiceEntryWrapper, 0), - toBeDeleted: make([]*ServiceEntryWrapper, 0), + sew: make(map[string]*ServiceWrapper), + toBeUpdated: make([]*ServiceWrapper, 0), + toBeDeleted: make([]*ServiceWrapper, 0), ip2services: make(map[string]map[string]bool), deferedDelete: make(map[string]struct{}), } @@ -51,9 +53,9 @@ func NewCache() Cache { type store struct { mux *sync.RWMutex - sew map[string]*ServiceEntryWrapper - toBeUpdated []*ServiceEntryWrapper - toBeDeleted []*ServiceEntryWrapper + sew map[string]*ServiceWrapper + toBeUpdated []*ServiceWrapper + toBeDeleted []*ServiceWrapper ip2services map[string]map[string]bool deferedDelete map[string]struct{} } @@ -94,7 +96,7 @@ func (s *store) UpdateServiceEntryEndpointWrapper(service, ip, regionId, zoneId, return } -func (s *store) UpdateServiceEntryWrapper(service string, data *ServiceEntryWrapper) { +func (s *store) UpdateServiceWrapper(service string, data *ServiceWrapper) { s.mux.Lock() defer s.mux.Unlock() @@ -116,7 +118,7 @@ func (s *store) UpdateServiceEntryWrapper(service string, data *ServiceEntryWrap log.Infof("ServiceEntry updated, host:%s", service) } -func (s *store) DeleteServiceEntryWrapper(service string) { +func (s *store) DeleteServiceWrapper(service string) { s.mux.Lock() defer s.mux.Unlock() @@ -199,31 +201,46 @@ func (s *store) GetAllServiceEntry() []*v1alpha3.ServiceEntry { return seList } -// GetAllServiceEntryWrapper get all ServiceEntryWrapper in the store for xds push -func (s *store) GetAllServiceEntryWrapper() []*ServiceEntryWrapper { +// GetAllServiceWrapper get all ServiceWrapper in the store for xds push +func (s *store) GetAllServiceWrapper() []*ServiceWrapper { s.mux.RLock() defer s.mux.RUnlock() defer s.cleanUpdateAndDeleteArray() - sewList := make([]*ServiceEntryWrapper, 0) + sewList := make([]*ServiceWrapper, 0) for _, serviceEntryWrapper := range s.sew { sewList = append(sewList, serviceEntryWrapper.DeepCopy()) } return sewList } -// GetIncrementalServiceEntryWrapper get incremental ServiceEntryWrapper in the store for xds push -func (s *store) GetIncrementalServiceEntryWrapper() ([]*ServiceEntryWrapper, []*ServiceEntryWrapper) { +// GetAllDestinationRuleWrapper get all DestinationRuleWrapper in the store for xds push +func (s *store) GetAllDestinationRuleWrapper() []*ingress.WrapperDestinationRule { s.mux.RLock() defer s.mux.RUnlock() defer s.cleanUpdateAndDeleteArray() - updatedList := make([]*ServiceEntryWrapper, 0) + drwList := make([]*ingress.WrapperDestinationRule, 0) + for _, serviceEntryWrapper := range s.sew { + if serviceEntryWrapper.DestinationRuleWrapper != nil { + drwList = append(drwList, serviceEntryWrapper.DeepCopy().DestinationRuleWrapper) + } + } + return drwList +} + +// GetIncrementalServiceWrapper get incremental ServiceWrapper in the store for xds push +func (s *store) GetIncrementalServiceWrapper() ([]*ServiceWrapper, []*ServiceWrapper) { + s.mux.RLock() + defer s.mux.RUnlock() + defer s.cleanUpdateAndDeleteArray() + + updatedList := make([]*ServiceWrapper, 0) for _, serviceEntryWrapper := range s.toBeUpdated { updatedList = append(updatedList, serviceEntryWrapper.DeepCopy()) } - deletedList := make([]*ServiceEntryWrapper, 0) + deletedList := make([]*ServiceWrapper, 0) for _, serviceEntryWrapper := range s.toBeDeleted { deletedList = append(deletedList, serviceEntryWrapper.DeepCopy()) } @@ -236,7 +253,7 @@ func (s *store) cleanUpdateAndDeleteArray() { s.toBeDeleted = nil } -func (s *store) updateIpMap(service string, data *ServiceEntryWrapper) { +func (s *store) updateIpMap(service string, data *ServiceWrapper) { for _, ep := range data.ServiceEntry.Endpoints { if s.ip2services[ep.Address] == nil { s.ip2services[ep.Address] = make(map[string]bool) diff --git a/registry/memory/model.go b/registry/memory/model.go index 3a3c3f7380..3d452209e0 100644 --- a/registry/memory/model.go +++ b/registry/memory/model.go @@ -18,27 +18,37 @@ import ( "time" "istio.io/api/networking/v1alpha3" + + "github.com/alibaba/higress/pkg/ingress/kube/common" ) -type ServiceEntryWrapper struct { - ServiceName string - ServiceEntry *v1alpha3.ServiceEntry - Suffix string - RegistryType string - createTime time.Time +type ServiceWrapper struct { + ServiceName string + ServiceEntry *v1alpha3.ServiceEntry + DestinationRuleWrapper *common.WrapperDestinationRule + Suffix string + RegistryType string + RegistryName string + createTime time.Time } -func (sew *ServiceEntryWrapper) DeepCopy() *ServiceEntryWrapper { - return &ServiceEntryWrapper{ - ServiceEntry: sew.ServiceEntry.DeepCopy(), - createTime: sew.GetCreateTime(), +func (sew *ServiceWrapper) DeepCopy() *ServiceWrapper { + res := &ServiceWrapper{} + res = sew + res.ServiceEntry = sew.ServiceEntry.DeepCopy() + res.createTime = sew.GetCreateTime() + + if sew.DestinationRuleWrapper != nil { + res.DestinationRuleWrapper = sew.DestinationRuleWrapper + res.DestinationRuleWrapper.DestinationRule = sew.DestinationRuleWrapper.DestinationRule.DeepCopy() } + return res } -func (sew *ServiceEntryWrapper) SetCreateTime(createTime time.Time) { +func (sew *ServiceWrapper) SetCreateTime(createTime time.Time) { sew.createTime = createTime } -func (sew *ServiceEntryWrapper) GetCreateTime() time.Time { +func (sew *ServiceWrapper) GetCreateTime() time.Time { return sew.createTime } diff --git a/registry/nacos/v2/watcher.go b/registry/nacos/v2/watcher.go index b51ed7d6e1..c58a092042 100644 --- a/registry/nacos/v2/watcher.go +++ b/registry/nacos/v2/watcher.go @@ -66,7 +66,7 @@ type watcher struct { isStop bool addrProvider *address.NacosAddressProvider updateCacheWhenEmpty bool - nacosClientConfig *constant.ClientConfig + nacosClientConfig *constant.ClientConfig authOption provider.AuthOption } @@ -413,7 +413,7 @@ func (w *watcher) getSubscribeCallback(groupName string, serviceName string) fun if err != nil { if strings.Contains(err.Error(), "hosts is empty") { if w.updateCacheWhenEmpty { - w.cache.DeleteServiceEntryWrapper(host) + w.cache.DeleteServiceWrapper(host) } } else { log.Errorf("callback error:%v", err) @@ -425,11 +425,12 @@ func (w *watcher) getSubscribeCallback(groupName string, serviceName string) fun return } serviceEntry := w.generateServiceEntry(host, services) - w.cache.UpdateServiceEntryWrapper(host, &memory.ServiceEntryWrapper{ + w.cache.UpdateServiceWrapper(host, &memory.ServiceWrapper{ ServiceName: serviceName, ServiceEntry: serviceEntry, Suffix: suffix, RegistryType: w.Type, + RegistryName: w.Name, }) } } @@ -487,7 +488,7 @@ func (w *watcher) Stop() { suffix := strings.Join([]string{s[0], w.NacosNamespace, "nacos"}, common.DotSeparator) suffix = strings.ReplaceAll(suffix, common.Underscore, common.Hyphen) host := strings.Join([]string{s[1], suffix}, common.DotSeparator) - w.cache.DeleteServiceEntryWrapper(host) + w.cache.DeleteServiceWrapper(host) } w.isStop = true diff --git a/registry/nacos/watcher.go b/registry/nacos/watcher.go index 08e30e82bd..132bcc0db9 100644 --- a/registry/nacos/watcher.go +++ b/registry/nacos/watcher.go @@ -301,7 +301,7 @@ func (w *watcher) getSubscribeCallback(groupName string, serviceName string) fun if err != nil { if strings.Contains(err.Error(), "hosts is empty") { if w.updateCacheWhenEmpty { - w.cache.DeleteServiceEntryWrapper(host) + w.cache.DeleteServiceWrapper(host) } } else { log.Errorf("callback error:%v", err) @@ -312,11 +312,12 @@ func (w *watcher) getSubscribeCallback(groupName string, serviceName string) fun return } serviceEntry := w.generateServiceEntry(host, services) - w.cache.UpdateServiceEntryWrapper(host, &memory.ServiceEntryWrapper{ + w.cache.UpdateServiceWrapper(host, &memory.ServiceWrapper{ ServiceName: serviceName, ServiceEntry: serviceEntry, Suffix: suffix, RegistryType: w.Type, + RegistryName: w.Name, }) } } @@ -374,7 +375,7 @@ func (w *watcher) Stop() { suffix := strings.Join([]string{s[0], w.NacosNamespace, w.Type}, common.DotSeparator) suffix = strings.ReplaceAll(suffix, common.Underscore, common.Hyphen) host := strings.Join([]string{s[1], suffix}, common.DotSeparator) - w.cache.DeleteServiceEntryWrapper(host) + w.cache.DeleteServiceWrapper(host) } w.isStop = true close(w.stop) diff --git a/registry/reconcile/reconcile.go b/registry/reconcile/reconcile.go index 84e304c381..21806d9e80 100644 --- a/registry/reconcile/reconcile.go +++ b/registry/reconcile/reconcile.go @@ -211,6 +211,8 @@ func (r *Reconciler) generateWatcherFromRegistryConfig(registry *apiv1.RegistryC direct.WithName(registry.Name), direct.WithDomain(registry.Domain), direct.WithPort(registry.Port), + direct.WithProtocol(registry.Protocol), + direct.WithSNI(registry.Sni), ) case string(Eureka): watcher, err = eureka.NewWatcher( diff --git a/registry/zookeeper/watcher.go b/registry/zookeeper/watcher.go index d90cd8385d..27bf3110b5 100644 --- a/registry/zookeeper/watcher.go +++ b/registry/zookeeper/watcher.go @@ -331,11 +331,12 @@ func (w *watcher) DataChange(eventType Event) bool { se := w.generateServiceEntry(w.serviceEntry[host]) w.seMux.Unlock() - w.cache.UpdateServiceEntryWrapper(host, &memory.ServiceEntryWrapper{ + w.cache.UpdateServiceWrapper(host, &memory.ServiceWrapper{ ServiceName: host, ServiceEntry: se, Suffix: "zookeeper", RegistryType: w.Type, + RegistryName: w.Name, }) w.UpdateService() } else if eventType.Action == EventTypeDel { @@ -358,14 +359,15 @@ func (w *watcher) DataChange(eventType Event) bool { //todo update if len(se.Endpoints) == 0 { if !w.keepStaleWhenEmpty { - w.cache.DeleteServiceEntryWrapper(host) + w.cache.DeleteServiceWrapper(host) } } else { - w.cache.UpdateServiceEntryWrapper(host, &memory.ServiceEntryWrapper{ + w.cache.UpdateServiceWrapper(host, &memory.ServiceWrapper{ ServiceName: host, ServiceEntry: se, Suffix: "zookeeper", RegistryType: w.Type, + RegistryName: w.Name, }) } w.UpdateService() @@ -560,20 +562,22 @@ func (w *watcher) ChildToServiceEntry(children []string, interfaceName, zkPath s if !reflect.DeepEqual(value, config) { w.serviceEntry[host] = config //todo update or create serviceentry - w.cache.UpdateServiceEntryWrapper(host, &memory.ServiceEntryWrapper{ + w.cache.UpdateServiceWrapper(host, &memory.ServiceWrapper{ ServiceName: host, ServiceEntry: se, Suffix: "zookeeper", RegistryType: w.Type, + RegistryName: w.Name, }) } } else { w.serviceEntry[host] = config - w.cache.UpdateServiceEntryWrapper(host, &memory.ServiceEntryWrapper{ + w.cache.UpdateServiceWrapper(host, &memory.ServiceWrapper{ ServiceName: host, ServiceEntry: se, Suffix: "zookeeper", RegistryType: w.Type, + RegistryName: w.Name, }) } } @@ -708,7 +712,7 @@ func (w *watcher) Stop() { w.seMux.Lock() for key := range w.serviceEntry { - w.cache.DeleteServiceEntryWrapper(key) + w.cache.DeleteServiceWrapper(key) } w.UpdateService() w.seMux.Unlock()