Merge branch 'master' into docs/observability

CHANGES.md

@@ -15,6 +15,16 @@ Apollo 2.4.0
 * [Feature add limit for items count per namespace](https://github.com/apolloconfig/apollo/pull/5227)
 * [Feature: Add ConfigService cache record stats function](https://github.com/apolloconfig/apollo/pull/5247)
 * [RefreshAdminServerAddressTask supports dynamic configuration of time interval](https://github.com/apolloconfig/apollo/pull/5248)
+* [Refactor: Configuration files uniformly use Kebab style](https://github.com/apolloconfig/apollo/pull/5262)
+* [Feature: openapi query namespace support not fill item](https://github.com/apolloconfig/apollo/pull/5249)
+* [Refactor: align database ClusterName and NamespaceName fields lengths](https://github.com/apolloconfig/apollo/pull/5263)
+* [Feature: Added the value length limit function for AppId-level configuration items](https://github.com/apolloconfig/apollo/pull/5264)
+* [Fix: ensure clusters order in envClusters open api](https://github.com/apolloconfig/apollo/pull/5277)
+* [Fix: bump xstream from 1.4.20 to 1.4.21 to fix CVE-2024-47072](https://github.com/apolloconfig/apollo/pull/5280)
+* [Feature: highlight diffs for properties](https://github.com/apolloconfig/apollo/pull/5282)
+* [Feature: Add rate limiting function to ConsumerToken](https://github.com/apolloconfig/apollo/pull/5267)
+* [Feature: add JSON formatting function in apollo-portal](https://github.com/apolloconfig/apollo/pull/5287)
+* [Fix: add missing url patterns for AdminServiceAuthenticationFilter](https://github.com/apolloconfig/apollo/pull/5291)
 
 ------------------
 All issues and pull requests are [here](https://github.com/apolloconfig/apollo/milestone/15?closed=1)

apollo-adminservice/src/main/java/com/ctrip/framework/apollo/adminservice/AdminServiceAutoConfiguration.java

@@ -36,12 +36,15 @@ public FilterRegistrationBean<AdminServiceAuthenticationFilter> adminServiceAuth
     FilterRegistrationBean<AdminServiceAuthenticationFilter> filterRegistrationBean = new FilterRegistrationBean<>();
 
     filterRegistrationBean.setFilter(new AdminServiceAuthenticationFilter(bizConfig));
+    filterRegistrationBean.addUrlPatterns("/apollo/audit/*");
     filterRegistrationBean.addUrlPatterns("/apps/*");
     filterRegistrationBean.addUrlPatterns("/appnamespaces/*");
     filterRegistrationBean.addUrlPatterns("/instances/*");
     filterRegistrationBean.addUrlPatterns("/items/*");
+    filterRegistrationBean.addUrlPatterns("/items-search/*");
     filterRegistrationBean.addUrlPatterns("/namespaces/*");
     filterRegistrationBean.addUrlPatterns("/releases/*");
+    filterRegistrationBean.addUrlPatterns("/server/*");
 
     return filterRegistrationBean;
   }

apollo-adminservice/src/main/resources/application-database-discovery.properties

@@ -18,5 +18,5 @@ spring.cloud.discovery.enabled=false
 
 apollo.service.registry.enabled=true
 apollo.service.registry.cluster=default
-apollo.service.registry.heartbeatIntervalInSecond=10
+apollo.service.registry.heartbeat-interval-in-second=10
 

apollo-adminservice/src/main/resources/application.yml

@@ -38,17 +38,17 @@ logging:
 eureka:
   instance:
     hostname: ${hostname:localhost}
-    preferIpAddress: true
+    prefer-ip-address: true
     status-page-url-path: /info
     health-check-url-path: /health
   client:
-    serviceUrl:
+    service-url:
       # This setting will be overridden by eureka.service.url setting from ApolloConfigDB.ServerConfig or System Property
       # see com.ctrip.framework.apollo.biz.eureka.ApolloEurekaClientConfig
       defaultZone: http://${eureka.instance.hostname}:8080/eureka/
     healthcheck:
       enabled: true
-    eurekaServiceUrlPollIntervalSeconds: 60
+    eureka-service-url-poll-interval-seconds: 60
 
 management:
   health:

apollo-adminservice/src/test/resources/application.yml

@@ -23,11 +23,11 @@ server:
 eureka:
   instance:
     hostname: ${hostname:localhost}
-    preferIpAddress: true
+    prefer-ip-address: true
     status-page-url-path: /info
     health-check-url-path: /health
   client:
-    serviceUrl:
+    service-url:
       defaultZone: http://${eureka.instance.hostname}:8090/eureka/
     healthcheck:
       enabled: true

apollo-assembly/src/main/resources/application-database-discovery.properties

@@ -19,8 +19,8 @@ spring.cloud.discovery.enabled=false
 
 apollo.service.registry.enabled=true
 apollo.service.registry.cluster=default
-apollo.service.registry.heartbeatIntervalInSecond=10
+apollo.service.registry.heartbeat-interval-in-second=10
 
 apollo.service.discovery.enabled=true
 # health check by heartbeat, heartbeat time before 61s ago will be seemed as unhealthy
-apollo.service.discovery.healthCheckIntervalInSecond = 61
+apollo.service.discovery.health-check-interval-in-second = 61

apollo-assembly/src/main/resources/application.yml

@@ -30,7 +30,7 @@ spring:
 
 logging:
   file:
-    name: /opt/logs/100003171/apollo-assembly.log
+    name: /opt/logs/apollo-assembly.log
 
 management:
   health:
@@ -42,22 +42,22 @@ management:
 eureka:
   instance:
     hostname: ${hostname:localhost}
-    preferIpAddress: true
+    prefer-ip-address: true
     status-page-url-path: /info
     health-check-url-path: /health
   server:
-    peerEurekaNodesUpdateIntervalMs: 60000
-    enableSelfPreservation: false
+    peer-eureka-nodes-update-interval-ms: 60000
+    enable-self-preservation: false
   client:
-    serviceUrl:
+    service-url:
       # This setting will be overridden by eureka.service.url setting from ApolloConfigDB.ServerConfig or System Property
       # see com.ctrip.framework.apollo.biz.eureka.ApolloEurekaClientConfig
       defaultZone: http://${eureka.instance.hostname}:8080/eureka/
     healthcheck:
       enabled: true
-    eurekaServiceUrlPollIntervalSeconds: 60
+    eureka-service-url-poll-interval-seconds: 60
     fetch-registry: false
-    registerWithEureka: false
+    register-with-eureka: false
 
 server:
   compression:

apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/config/BizConfig.java

@@ -30,12 +30,16 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
+import java.util.function.Predicate;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Component;
 
 @Component
 public class BizConfig extends RefreshableConfig {
 
+  private final static Logger logger = LoggerFactory.getLogger(BizConfig.class);
+
   private static final int DEFAULT_ITEM_KEY_LENGTH = 128;
   private static final int DEFAULT_ITEM_VALUE_LENGTH = 20000;
 
@@ -58,6 +62,9 @@ public class BizConfig extends RefreshableConfig {
 
   private static final Gson GSON = new Gson();
 
+  private static final Type appIdValueLengthOverrideTypeReference =
+      new TypeToken<Map<String, Integer>>() {
+      }.getType();
   private static final Type namespaceValueLengthOverrideTypeReference =
       new TypeToken<Map<Long, Integer>>() {
       }.getType();
@@ -106,6 +113,16 @@ public int itemValueLengthLimit() {
     return checkInt(limit, 5, Integer.MAX_VALUE, DEFAULT_ITEM_VALUE_LENGTH);
   }
 
+  public Map<String, Integer> appIdValueLengthLimitOverride() {
+    String appIdValueLengthOverrideString = getValue("appid.value.length.limit.override");
+    return parseOverrideConfig(appIdValueLengthOverrideString, appIdValueLengthOverrideTypeReference, value -> value > 0);
+  }
+
+  public Map<Long, Integer> namespaceValueLengthLimitOverride() {
+    String namespaceValueLengthOverrideString = getValue("namespace.value.length.limit.override");
+    return parseOverrideConfig(namespaceValueLengthOverrideString, namespaceValueLengthOverrideTypeReference, value -> value > 0);
+  }
+
   public boolean isNamespaceNumLimitEnabled() {
     return getBooleanProperty("namespace.num.limit.enabled", false);
   }
@@ -128,17 +145,6 @@ public int itemNumLimit() {
     return checkInt(limit, 5, Integer.MAX_VALUE, DEFAULT_MAX_ITEM_NUM);
   }
 
-  public Map<Long, Integer> namespaceValueLengthLimitOverride() {
-    String namespaceValueLengthOverrideString = getValue("namespace.value.length.limit.override");
-    Map<Long, Integer> namespaceValueLengthOverride = Maps.newHashMap();
-    if (!Strings.isNullOrEmpty(namespaceValueLengthOverrideString)) {
-      namespaceValueLengthOverride =
-          GSON.fromJson(namespaceValueLengthOverrideString, namespaceValueLengthOverrideTypeReference);
-    }
-
-    return namespaceValueLengthOverride;
-  }
-
   public boolean isNamespaceLockSwitchOff() {
     return !getBooleanProperty("namespace.lock.switch", false);
   }
@@ -195,15 +201,7 @@ public int releaseHistoryRetentionSize() {
 
   public Map<String, Integer> releaseHistoryRetentionSizeOverride() {
     String overrideString = getValue("apollo.release-history.retention.size.override");
-    Map<String, Integer> releaseHistoryRetentionSizeOverride = Maps.newHashMap();
-    if (!Strings.isNullOrEmpty(overrideString)) {
-      releaseHistoryRetentionSizeOverride =
-          GSON.fromJson(overrideString, releaseHistoryRetentionSizeOverrideTypeReference);
-    }
-    return releaseHistoryRetentionSizeOverride.entrySet()
-        .stream()
-        .filter(entry -> entry.getValue() >= 1)
-        .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+    return parseOverrideConfig(overrideString, releaseHistoryRetentionSizeOverrideTypeReference, value -> value > 0);
   }
 
   public int releaseMessageCacheScanInterval() {
@@ -256,4 +254,22 @@ public boolean isAdminServiceAccessControlEnabled() {
   public String getAdminServiceAccessTokens() {
     return getValue("admin-service.access.tokens");
   }
+
+  private <K, V> Map<K, V> parseOverrideConfig(String configValue, Type typeReference, Predicate<V> valueFilter) {
+    Map<K, V> result = Maps.newHashMap();
+    if (!Strings.isNullOrEmpty(configValue)) {
+      try {
+        Map<K, V> parsed = GSON.fromJson(configValue, typeReference);
+        for (Map.Entry<K, V> entry : parsed.entrySet()) {
+          if (entry.getValue() != null && valueFilter.test(entry.getValue())) {
+            result.put(entry.getKey(), entry.getValue());
+          }
+        }
+      } catch (Exception e) {
+        logger.error("Invalid override config value: {}", configValue, e);
+      }
+    }
+    return Collections.unmodifiableMap(result);
+  }
+
 }

apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/service/ItemService.java

@@ -217,8 +217,8 @@ public Item update(Item item) {
   }
 
   private boolean checkItemValueLength(long namespaceId, String value) {
-    int limit = getItemValueLengthLimit(namespaceId);
     Namespace currentNamespace = namespaceService.findOne(namespaceId);
+    int limit = getItemValueLengthLimit(currentNamespace);
     if(currentNamespace != null) {
       Matcher m = clusterPattern.matcher(currentNamespace.getClusterName());
       boolean isGray = m.matches();
@@ -235,7 +235,7 @@ private boolean checkItemValueLength(long namespaceId, String value) {
   private int getGrayNamespaceItemValueLengthLimit(Namespace grayNamespace, int grayNamespaceLimit) {
     Namespace parentNamespace = namespaceService.findParentNamespace(grayNamespace);
     if (parentNamespace != null) {
-      int parentLimit = getItemValueLengthLimit(parentNamespace.getId());
+      int parentLimit = getItemValueLengthLimit(grayNamespace);
       if (parentLimit > grayNamespaceLimit) {
         return parentLimit;
       }
@@ -257,11 +257,17 @@ private boolean checkItemType(int type) {
     return true;
   }
 
-  private int getItemValueLengthLimit(long namespaceId) {
+  private int getItemValueLengthLimit(Namespace namespace) {
     Map<Long, Integer> namespaceValueLengthOverride = bizConfig.namespaceValueLengthLimitOverride();
-    if (namespaceValueLengthOverride != null && namespaceValueLengthOverride.containsKey(namespaceId)) {
-      return namespaceValueLengthOverride.get(namespaceId);
+    if (namespaceValueLengthOverride != null && namespaceValueLengthOverride.containsKey(namespace.getId())) {
+      return namespaceValueLengthOverride.get(namespace.getId());
     }
+
+    Map<String, Integer> appIdValueLengthOverride = bizConfig.appIdValueLengthLimitOverride();
+    if (appIdValueLengthOverride != null && appIdValueLengthOverride.containsKey(namespace.getAppId())) {
+      return appIdValueLengthOverride.get(namespace.getAppId());
+    }
+
     return bizConfig.itemValueLengthLimit();
   }
 

apollo-biz/src/test/java/com/ctrip/framework/apollo/biz/config/BizConfigTest.java

@@ -18,13 +18,13 @@
 
 import com.ctrip.framework.apollo.biz.repository.ServerConfigRepository;
 import com.ctrip.framework.apollo.biz.service.BizDBPropertySource;
+import java.util.Map;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.mockito.Mock;
 import org.mockito.junit.MockitoJUnitRunner;
 import org.springframework.core.env.ConfigurableEnvironment;
-import org.springframework.core.env.Environment;
 import org.springframework.test.util.ReflectionTestUtils;
 
 import javax.sql.DataSource;
@@ -93,7 +93,7 @@ public void testReleaseHistoryRetentionSizeOverride() {
     int someOverrideLimit = 10;
     String overrideValueString = "{'a+b+c+b':10}";
     when(environment.getProperty("apollo.release-history.retention.size.override")).thenReturn(overrideValueString);
-    int  overrideValue = bizConfig.releaseHistoryRetentionSizeOverride().get("a+b+c+b");
+    int overrideValue = bizConfig.releaseHistoryRetentionSizeOverride().get("a+b+c+b");
     assertEquals(someOverrideLimit, overrideValue);
 
     overrideValueString = "{'a+b+c+b':0,'a+b+d+b':2}";
@@ -107,6 +107,48 @@ public void testReleaseHistoryRetentionSizeOverride() {
     assertEquals(0, bizConfig.releaseHistoryRetentionSizeOverride().size());
   }
 
+  @Test
+  public void testAppIdValueLengthLimitOverride() {
+    when(environment.getProperty("appid.value.length.limit.override")).thenReturn(null);
+    Map<String, Integer> result = bizConfig.appIdValueLengthLimitOverride();
+    assertTrue(result.isEmpty());
+
+    String input = "{}";
+    when(environment.getProperty("appid.value.length.limit.override")).thenReturn(input);
+    result = bizConfig.appIdValueLengthLimitOverride();
+    assertTrue(result.isEmpty());
+
+    input = "invalid json";
+    when(environment.getProperty("appid.value.length.limit.override")).thenReturn(input);
+    result = bizConfig.appIdValueLengthLimitOverride();
+    assertTrue(result.isEmpty());
+
+    input = "{'appid1':555}";
+    when(environment.getProperty("appid.value.length.limit.override")).thenReturn(input);
+    int overrideValue = bizConfig.appIdValueLengthLimitOverride().get("appid1");
+    assertEquals(1, bizConfig.appIdValueLengthLimitOverride().size());
+    assertEquals(555, overrideValue);
+
+    input = "{'appid1':555,'appid2':666}";
+    when(environment.getProperty("appid.value.length.limit.override")).thenReturn(input);
+    overrideValue = bizConfig.appIdValueLengthLimitOverride().get("appid2");
+    assertEquals(2, bizConfig.appIdValueLengthLimitOverride().size());
+    assertEquals(666, overrideValue);
+
+    input = "{'appid1':555,'appid2':666,'appid3':0,'appid4':-1}";
+    when(environment.getProperty("appid.value.length.limit.override")).thenReturn(input);
+    result = bizConfig.appIdValueLengthLimitOverride();
+
+    assertTrue(result.containsKey("appid1"));
+    assertTrue(result.containsKey("appid2"));
+    assertFalse(result.containsKey("appid3"));
+    assertFalse(result.containsKey("appid4"));
+    assertEquals(2, result.size());
+
+    overrideValue = result.get("appid2");
+    assertEquals(666, overrideValue);
+  }
+
   @Test
   public void testReleaseMessageNotificationBatchWithNAN() throws Exception {
     String someNAN = "someNAN";