config envoy local cluster and isolate apiToken ctx between different…

… providers

helm/core/templates/_pod.tpl

@@ -123,10 +123,8 @@ template:
         - name: LITE_METRICS
           value: "on"
         {{- end }}
-        {{- if include "skywalking.enabled" . }}
         - name: ISTIO_BOOTSTRAP_OVERRIDE
           value: /etc/istio/custom-bootstrap/custom_bootstrap.json
-        {{- end }}
         {{- with .Values.gateway.networkGateway }}
         - name: ISTIO_META_REQUESTED_NETWORK_VIEW
           value: "{{.}}"
@@ -188,10 +186,8 @@ template:
           mountPath: /etc/istio/pod
         - name: proxy-socket
           mountPath: /etc/istio/proxy
-        {{- if include "skywalking.enabled" . }}
         - mountPath: /etc/istio/custom-bootstrap
           name: custom-bootstrap-volume
-        {{- end }}
         {{- if .Values.global.volumeWasmPlugins }}
         - mountPath: /opt/plugins
           name: local-wasmplugins-volume
@@ -276,12 +272,10 @@ template:
     - name: config
       configMap:
         name: higress-config
-    {{- if include "skywalking.enabled" . }}
     - configMap:
         defaultMode: 420
         name: higress-custom-bootstrap
       name: custom-bootstrap-volume
-    {{- end }}
     - name: istio-data
       emptyDir: {}
     - name: proxy-socket

helm/core/templates/configmap.yaml

@@ -136,7 +136,6 @@ data:
 {{- include "mesh" . }}
 {{- end }}
 ---
-{{- if include "skywalking.enabled" . }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -147,6 +146,7 @@ metadata:
 data:
   custom_bootstrap.json: |-
     {
+      {{- if include "skywalking.enabled" . }}
       "stats_sinks": [
         {
           "name": "envoy.metrics_service",
@@ -161,8 +161,10 @@ data:
           }
         }
       ],
+      {{- end }}
       "static_resources": {
         "clusters": [
+          {{- if include "skywalking.enabled" . }}
           {
             "name": "service_skywalking",
             "type": "LOGICAL_DNS",
@@ -190,9 +192,32 @@ data:
                 }
               ]
             }
+          },
+          {{- end }}
+          {
+            "name": "higress-gateway-local",
+            "type": "STATIC",
+            "connect_timeout": "5s",
+            "load_assignment": {
+              "cluster_name": "higress-gateway-local",
+              "endpoints": [
+                {
+                  "lb_endpoints": [
+                    {
+                      "endpoint": {
+                        "address": {
+                          "socket_address": {
+                            "address": "127.0.0.1",
+                            "port_value": {{ .Values.gateway.httpPort }}
+                          }
+                        }
+                      }
+                    }
+                  ]
+                }
+              ]
+            }
           }
         ]
       }
     }
----
-{{- end }}

plugins/wasm-go/extensions/ai-proxy/config/config.go

@@ -80,14 +80,12 @@ func (c *PluginConfig) Complete(log wrapper.Log) error {
 		c.activeProvider = nil
 		return nil
 	}
+	var err error
+	c.activeProvider, err = provider.CreateProvider(*c.activeProviderConfig)
 
 	providerConfig := c.GetProviderConfig()
-	err := providerConfig.SetApiTokensFailover(log)
-	if err != nil {
-		return err
-	}
+	err = providerConfig.SetApiTokensFailover(log)
 
-	c.activeProvider, err = provider.CreateProvider(*c.activeProviderConfig)
 	return err
 }
 

plugins/wasm-go/extensions/ai-proxy/main.go

@@ -94,6 +94,8 @@ func onHttpRequestHeader(ctx wrapper.HttpContext, pluginConfig config.PluginConf
 		ctx.DisableReroute()
 		// Set the apiToken for the current request.
 		providerConfig.SetApiTokenInUse(ctx, log)
+		// Set the request host and path to shared data in case they are needed in apiToken health check
+		providerConfig.SetRequestHostAndPath(log)
 
 		hasRequestBody := wrapper.HasRequestBody()
 		action, err := handler.OnRequestHeaders(ctx, apiName, log)

plugins/wasm-go/extensions/ai-proxy/provider/ai360.go

@@ -49,7 +49,7 @@ func (m *ai360Provider) OnRequestHeaders(ctx wrapper.HttpContext, apiName ApiNam
 	_ = util.OverwriteRequestHost(ai360Domain)
 	_ = proxywasm.RemoveHttpRequestHeader("Accept-Encoding")
 	_ = proxywasm.RemoveHttpRequestHeader("Content-Length")
-	_ = proxywasm.ReplaceHttpRequestHeader("Authorization", getApiTokenInUse(ctx))
+	_ = proxywasm.ReplaceHttpRequestHeader("Authorization", m.config.GetApiTokenInUse(ctx))
 	// Delay the header processing to allow changing streaming mode in OnRequestBody
 	return types.HeaderStopIteration, nil
 }

plugins/wasm-go/extensions/ai-proxy/provider/azure.go

@@ -57,7 +57,7 @@ func (m *azureProvider) GetProviderType() string {
 func (m *azureProvider) OnRequestHeaders(ctx wrapper.HttpContext, apiName ApiName, log wrapper.Log) (types.Action, error) {
 	_ = util.OverwriteRequestPath(m.serviceUrl.RequestURI())
 	_ = util.OverwriteRequestHost(m.serviceUrl.Host)
-	_ = proxywasm.ReplaceHttpRequestHeader("api-key", m.config.apiTokens[0])
+	_ = proxywasm.ReplaceHttpRequestHeader("api-key", m.config.GetApiTokenInUse(ctx))
 	if apiName == ApiNameChatCompletion {
 		_ = proxywasm.RemoveHttpRequestHeader("Content-Length")
 	} else {

plugins/wasm-go/extensions/ai-proxy/provider/baichuan.go

@@ -49,7 +49,7 @@ func (m *baichuanProvider) OnRequestHeaders(ctx wrapper.HttpContext, apiName Api
 	}
 	_ = util.OverwriteRequestPath(baichuanChatCompletionPath)
 	_ = util.OverwriteRequestHost(baichuanDomain)
-	_ = util.OverwriteRequestAuthorization("Bearer " + getApiTokenInUse(ctx))
+	_ = util.OverwriteRequestAuthorization("Bearer " + m.config.GetApiTokenInUse(ctx))
 	_ = proxywasm.RemoveHttpRequestHeader("Content-Length")
 	return types.ActionContinue, nil
 }

plugins/wasm-go/extensions/ai-proxy/provider/baidu.go

@@ -232,7 +232,7 @@ func (b *baiduProvider) getRequestPath(ctx wrapper.HttpContext, baiduModel strin
 	if !ok {
 		suffix = baiduModel
 	}
-	return fmt.Sprintf("/rpc/2.0/ai_custom/v1/wenxinworkshop/chat/%s?access_token=%s", suffix, getApiTokenInUse(ctx))
+	return fmt.Sprintf("/rpc/2.0/ai_custom/v1/wenxinworkshop/chat/%s?access_token=%s", suffix, b.config.GetApiTokenInUse(ctx))
 }
 
 func (b *baiduProvider) setSystemContent(request *baiduTextGenRequest, content string) {

plugins/wasm-go/extensions/ai-proxy/provider/claude.go

@@ -108,7 +108,7 @@ func (c *claudeProvider) OnRequestHeaders(ctx wrapper.HttpContext, apiName ApiNa
 
 	_ = util.OverwriteRequestPath(claudeChatCompletionPath)
 	_ = util.OverwriteRequestHost(claudeDomain)
-	_ = proxywasm.ReplaceHttpRequestHeader("x-api-key", getApiTokenInUse(ctx))
+	_ = proxywasm.ReplaceHttpRequestHeader("x-api-key", c.config.GetApiTokenInUse(ctx))
 
 	if c.config.claudeVersion == "" {
 		c.config.claudeVersion = defaultVersion

plugins/wasm-go/extensions/ai-proxy/provider/cloudflare.go

@@ -49,7 +49,7 @@ func (c *cloudflareProvider) OnRequestHeaders(ctx wrapper.HttpContext, apiName A
 	}
 	_ = util.OverwriteRequestPath(strings.Replace(cloudflareChatCompletionPath, "{account_id}", c.config.cloudflareAccountId, 1))
 	_ = util.OverwriteRequestHost(cloudflareDomain)
-	_ = util.OverwriteRequestAuthorization("Bearer " + getApiTokenInUse(ctx))
+	_ = util.OverwriteRequestAuthorization("Bearer " + c.config.GetApiTokenInUse(ctx))
 
 	_ = proxywasm.RemoveHttpRequestHeader("Accept-Encoding")
 	_ = proxywasm.RemoveHttpRequestHeader("Content-Length")

plugins/wasm-go/extensions/ai-proxy/provider/cohere.go

@@ -59,7 +59,7 @@ func (m *cohereProvider) OnRequestHeaders(ctx wrapper.HttpContext, apiName ApiNa
 	}
 	_ = util.OverwriteRequestHost(cohereDomain)
 	_ = util.OverwriteRequestPath(chatCompletionPath)
-	_ = util.OverwriteRequestAuthorization("Bearer " + getApiTokenInUse(ctx))
+	_ = util.OverwriteRequestAuthorization("Bearer " + m.config.GetApiTokenInUse(ctx))
 	_ = proxywasm.RemoveHttpRequestHeader("Content-Length")
 	return types.ActionContinue, nil
 }

plugins/wasm-go/extensions/ai-proxy/provider/deepl.go

@@ -79,7 +79,7 @@ func (d *deeplProvider) OnRequestHeaders(ctx wrapper.HttpContext, apiName ApiNam
 		return types.ActionContinue, errUnsupportedApiName
 	}
 	_ = util.OverwriteRequestPath(deeplChatCompletionPath)
-	_ = util.OverwriteRequestAuthorization("DeepL-Auth-Key " + getApiTokenInUse(ctx))
+	_ = util.OverwriteRequestAuthorization("DeepL-Auth-Key " + d.config.GetApiTokenInUse(ctx))
 	_ = proxywasm.RemoveHttpRequestHeader("Content-Length")
 	_ = proxywasm.RemoveHttpRequestHeader("Accept-Encoding")
 	return types.HeaderStopIteration, nil

plugins/wasm-go/extensions/ai-proxy/provider/deepseek.go

@@ -49,7 +49,7 @@ func (m *deepseekProvider) OnRequestHeaders(ctx wrapper.HttpContext, apiName Api
 	}
 	_ = util.OverwriteRequestPath(deepseekChatCompletionPath)
 	_ = util.OverwriteRequestHost(deepseekDomain)
-	_ = util.OverwriteRequestAuthorization("Bearer " + getApiTokenInUse(ctx))
+	_ = util.OverwriteRequestAuthorization("Bearer " + m.config.GetApiTokenInUse(ctx))
 	_ = proxywasm.RemoveHttpRequestHeader("Content-Length")
 	return types.ActionContinue, nil
 }

plugins/wasm-go/extensions/ai-proxy/provider/doubao.go

@@ -42,7 +42,7 @@ func (m *doubaoProvider) GetProviderType() string {
 
 func (m *doubaoProvider) OnRequestHeaders(ctx wrapper.HttpContext, apiName ApiName, log wrapper.Log) (types.Action, error) {
 	_ = util.OverwriteRequestHost(doubaoDomain)
-	_ = util.OverwriteRequestAuthorization("Bearer " + getApiTokenInUse(ctx))
+	_ = util.OverwriteRequestAuthorization("Bearer " + m.config.GetApiTokenInUse(ctx))
 	_ = proxywasm.RemoveHttpRequestHeader("Content-Length")
 	if m.config.protocol == protocolOriginal {
 		ctx.DontReadRequestBody()