diff --git a/documentation/compose/kafka-ui-arm64.yaml b/documentation/compose/kafka-ui-arm64.yaml index 082d7cb5af0..91f8dc252ce 100644 --- a/documentation/compose/kafka-ui-arm64.yaml +++ b/documentation/compose/kafka-ui-arm64.yaml @@ -13,16 +13,29 @@ services: - schema-registry0 - kafka-connect0 environment: + DYNAMIC_CONFIG_ENABLED: 'true' # not necessary, added for tests KAFKA_CLUSTERS_0_NAME: local KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka0:29092 KAFKA_CLUSTERS_0_METRICS_PORT: 9997 + KAFKA_CLUSTERS_0_METRICS_STORE_PROMETHEUS_URL: "http://prometheus:9090" + KAFKA_CLUSTERS_0_METRICS_STORE_PROMETHEUS_REMOTEWRITE: 'true' + KAFKA_CLUSTERS_0_METRICS_STORE_KAFKA_TOPIC: "kafka_metrics" KAFKA_CLUSTERS_0_SCHEMAREGISTRY: http://schema-registry0:8085 KAFKA_CLUSTERS_0_KAFKACONNECT_0_NAME: first KAFKA_CLUSTERS_0_KAFKACONNECT_0_ADDRESS: http://kafka-connect0:8083 - DYNAMIC_CONFIG_ENABLED: 'true' # not necessary, added for tests KAFKA_CLUSTERS_0_AUDIT_TOPICAUDITENABLED: 'true' KAFKA_CLUSTERS_0_AUDIT_CONSOLEAUDITENABLED: 'true' + prometheus: + image: prom/prometheus:latest + hostname: prometheus + container_name: prometheus + ports: + - 9090:9090 + volumes: + - ./scripts:/etc/prometheus + command: --web.enable-remote-write-receiver --config.file=/etc/prometheus/prometheus.yaml + kafka0: image: confluentinc/cp-kafka:7.2.1.arm64 hostname: kafka0 diff --git a/documentation/compose/scripts/prometheus.yaml b/documentation/compose/scripts/prometheus.yaml new file mode 100644 index 00000000000..457de126cd5 --- /dev/null +++ b/documentation/compose/scripts/prometheus.yaml @@ -0,0 +1,14 @@ +global: + scrape_interval: 30s + scrape_timeout: 10s + +rule_files: + - alert.yml + +scrape_configs: + - job_name: services + metrics_path: /metrics + static_configs: + - targets: + - 'prometheus:9090' +# - 'kafka-ui:8080' diff --git a/kafka-ui-api/pom.xml b/kafka-ui-api/pom.xml index 558a446e948..a964a5ebe74 100644 --- a/kafka-ui-api/pom.xml +++ b/kafka-ui-api/pom.xml @@ -239,6 +239,23 @@ spring-security-ldap + + io.prometheus + simpleclient + + + io.prometheus + simpleclient_common + + + io.prometheus + simpleclient_pushgateway + + + org.xerial.snappy + snappy-java + 1.1.8.4 + org.codehaus.groovy diff --git a/kafka-ui-api/src/main/antlr4/promql/PromQLLexer.g4 b/kafka-ui-api/src/main/antlr4/promql/PromQLLexer.g4 new file mode 100644 index 00000000000..9cda649422a --- /dev/null +++ b/kafka-ui-api/src/main/antlr4/promql/PromQLLexer.g4 @@ -0,0 +1,176 @@ +lexer grammar PromQLLexer; + +channels { WHITESPACE, COMMENTS } + +// All keywords in PromQL are case insensitive, it is just function, +// label and metric names that are not. +options { caseInsensitive=true; } + +fragment NUMERAL: [0-9]+ ('.' [0-9]+)?; + +fragment SCIENTIFIC_NUMBER + : NUMERAL ('e' [-+]? NUMERAL)? + ; + +NUMBER + : NUMERAL + | SCIENTIFIC_NUMBER; + +STRING + : '\'' (~('\'' | '\\') | '\\' .)* '\'' + | '"' (~('"' | '\\') | '\\' .)* '"' + ; + +// Binary operators + +ADD: '+'; +SUB: '-'; +MULT: '*'; +DIV: '/'; +MOD: '%'; +POW: '^'; + +AND: 'and'; +OR: 'or'; +UNLESS: 'unless'; + +// Comparison operators + +EQ: '='; +DEQ: '=='; +NE: '!='; +GT: '>'; +LT: '<'; +GE: '>='; +LE: '<='; +RE: '=~'; +NRE: '!~'; + +// Aggregation modifiers + +BY: 'by'; +WITHOUT: 'without'; + +// Join modifiers + +ON: 'on'; +IGNORING: 'ignoring'; +GROUP_LEFT: 'group_left'; +GROUP_RIGHT: 'group_right'; + +OFFSET: 'offset'; + +BOOL: 'bool'; + +AGGREGATION_OPERATOR + : 'sum' + | 'min' + | 'max' + | 'avg' + | 'group' + | 'stddev' + | 'stdvar' + | 'count' + | 'count_values' + | 'bottomk' + | 'topk' + | 'quantile' + ; + +FUNCTION options { caseInsensitive=false; } + : 'abs' + | 'absent' + | 'absent_over_time' + | 'ceil' + | 'changes' + | 'clamp_max' + | 'clamp_min' + | 'day_of_month' + | 'day_of_week' + | 'days_in_month' + | 'delta' + | 'deriv' + | 'exp' + | 'floor' + | 'histogram_quantile' + | 'holt_winters' + | 'hour' + | 'idelta' + | 'increase' + | 'irate' + | 'label_join' + | 'label_replace' + | 'ln' + | 'log2' + | 'log10' + | 'minute' + | 'month' + | 'predict_linear' + | 'rate' + | 'resets' + | 'round' + | 'scalar' + | 'sort' + | 'sort_desc' + | 'sqrt' + | 'time' + | 'timestamp' + | 'vector' + | 'year' + | 'avg_over_time' + | 'min_over_time' + | 'max_over_time' + | 'sum_over_time' + | 'count_over_time' + | 'quantile_over_time' + | 'stddev_over_time' + | 'stdvar_over_time' + | 'last_over_time' + | 'acos' + | 'acosh' + | 'asin' + | 'asinh' + | 'atan' + | 'atanh' + | 'cos' + | 'cosh' + | 'sin' + | 'sinh' + | 'tan' + | 'tanh' + | 'deg' + | 'pi' + | 'rad' + ; + +LEFT_BRACE: '{'; +RIGHT_BRACE: '}'; + +LEFT_PAREN: '('; +RIGHT_PAREN: ')'; + +LEFT_BRACKET: '['; +RIGHT_BRACKET: ']'; + +COMMA: ','; + +AT: '@'; + +SUBQUERY_RANGE + : LEFT_BRACKET DURATION ':' DURATION? RIGHT_BRACKET; + +TIME_RANGE + : LEFT_BRACKET DURATION RIGHT_BRACKET; + +// The proper order (longest to the shortest) must be validated after parsing +DURATION: ([0-9]+ ('ms' | [smhdwy]))+; + +METRIC_NAME: [a-z_:] [a-z0-9_:]*; +LABEL_NAME: [a-z_] [a-z0-9_]*; + + + +WS: [\r\t\n ]+ -> channel(WHITESPACE); +SL_COMMENT + : '#' .*? '\n' -> channel(COMMENTS) + ; diff --git a/kafka-ui-api/src/main/antlr4/promql/PromQLParser.g4 b/kafka-ui-api/src/main/antlr4/promql/PromQLParser.g4 new file mode 100644 index 00000000000..c7703b709fc --- /dev/null +++ b/kafka-ui-api/src/main/antlr4/promql/PromQLParser.g4 @@ -0,0 +1,114 @@ +parser grammar PromQLParser; + +options { tokenVocab = PromQLLexer; } + +expression: vectorOperation EOF; + +// Binary operations are ordered by precedence + +// Unary operations have the same precedence as multiplications + +vectorOperation + : vectorOperation powOp vectorOperation + | vectorOperation subqueryOp + | unaryOp vectorOperation + | vectorOperation multOp vectorOperation + | vectorOperation addOp vectorOperation + | vectorOperation compareOp vectorOperation + | vectorOperation andUnlessOp vectorOperation + | vectorOperation orOp vectorOperation + | vectorOperation vectorMatchOp vectorOperation + | vectorOperation AT vectorOperation + | vector + ; + +// Operators + +unaryOp: (ADD | SUB); +powOp: POW grouping?; +multOp: (MULT | DIV | MOD) grouping?; +addOp: (ADD | SUB) grouping?; +compareOp: (DEQ | NE | GT | LT | GE | LE) BOOL? grouping?; +andUnlessOp: (AND | UNLESS) grouping?; +orOp: OR grouping?; +vectorMatchOp: (ON | UNLESS) grouping?; +subqueryOp: SUBQUERY_RANGE offsetOp?; +offsetOp: OFFSET DURATION; + +vector + : function_ + | aggregation + | instantSelector + | matrixSelector + | offset + | literal + | parens + ; + +parens: LEFT_PAREN vectorOperation RIGHT_PAREN; + +// Selectors + +instantSelector + : METRIC_NAME (LEFT_BRACE labelMatcherList? RIGHT_BRACE)? + | LEFT_BRACE labelMatcherList RIGHT_BRACE + ; + +labelMatcher: labelName labelMatcherOperator STRING; +labelMatcherOperator: EQ | NE | RE | NRE; +labelMatcherList: labelMatcher (COMMA labelMatcher)* COMMA?; + +matrixSelector: instantSelector TIME_RANGE; + +offset + : instantSelector OFFSET DURATION + | matrixSelector OFFSET DURATION + ; + +// Functions + +function_: FUNCTION LEFT_PAREN (parameter (COMMA parameter)*)? RIGHT_PAREN; + +parameter: literal | vectorOperation; +parameterList: LEFT_PAREN (parameter (COMMA parameter)*)? RIGHT_PAREN; + +// Aggregations + +aggregation + : AGGREGATION_OPERATOR parameterList + | AGGREGATION_OPERATOR (by | without) parameterList + | AGGREGATION_OPERATOR parameterList ( by | without) + ; +by: BY labelNameList; +without: WITHOUT labelNameList; + +// Vector one-to-one/one-to-many joins + +grouping: (on_ | ignoring) (groupLeft | groupRight)?; +on_: ON labelNameList; +ignoring: IGNORING labelNameList; +groupLeft: GROUP_LEFT labelNameList?; +groupRight: GROUP_RIGHT labelNameList?; + +// Label names + +labelName: keyword | METRIC_NAME | LABEL_NAME; +labelNameList: LEFT_PAREN (labelName (COMMA labelName)*)? RIGHT_PAREN; + +keyword + : AND + | OR + | UNLESS + | BY + | WITHOUT + | ON + | IGNORING + | GROUP_LEFT + | GROUP_RIGHT + | OFFSET + | BOOL + | AGGREGATION_OPERATOR + | FUNCTION + ; + +literal: NUMBER | STRING; diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/ClustersProperties.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/ClustersProperties.java index e0b20d6c93f..ed91fc35151 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/ClustersProperties.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/ClustersProperties.java @@ -1,6 +1,7 @@ package com.provectus.kafka.ui.config; -import com.provectus.kafka.ui.model.MetricsConfig; +import static com.provectus.kafka.ui.model.MetricsScrapeProperties.JMX_METRICS_TYPE; + import jakarta.annotation.PostConstruct; import java.util.ArrayList; import java.util.HashMap; @@ -42,7 +43,7 @@ public static class Cluster { KsqldbServerAuth ksqldbServerAuth; KeystoreConfig ksqldbServerSsl; List kafkaConnect; - MetricsConfigData metrics; + MetricsConfig metrics; Map properties; boolean readOnly = false; List serde; @@ -62,8 +63,8 @@ public static class PollingProperties { } @Data - @ToString(exclude = "password") - public static class MetricsConfigData { + @ToString(exclude = {"password", "keystorePassword"}) + public static class MetricsConfig { String type; Integer port; Boolean ssl; @@ -71,6 +72,31 @@ public static class MetricsConfigData { String password; String keystoreLocation; String keystorePassword; + + Boolean prometheusExpose; + MetricsStorage store; + } + + @Data + public static class MetricsStorage { + PrometheusStorage prometheus; + KafkaMetricsStorage kafka; + } + + @Data + public static class KafkaMetricsStorage { + String topic; + } + + @Data + @ToString(exclude = {"pushGatewayPassword"}) + public static class PrometheusStorage { + String url; + String pushGatewayUrl; + String pushGatewayUsername; + String pushGatewayPassword; + String pushGatewayJobName; + Boolean remoteWrite; } @Data @@ -171,7 +197,7 @@ public void validateAndSetDefaults() { private void setMetricsDefaults() { for (Cluster cluster : clusters) { if (cluster.getMetrics() != null && !StringUtils.hasText(cluster.getMetrics().getType())) { - cluster.getMetrics().setType(MetricsConfig.JMX_METRICS_TYPE); + cluster.getMetrics().setType(JMX_METRICS_TYPE); } } } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/AbstractAuthSecurityConfig.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/AbstractAuthSecurityConfig.java index 0c70b79716e..70807295cb4 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/AbstractAuthSecurityConfig.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/AbstractAuthSecurityConfig.java @@ -18,7 +18,8 @@ protected AbstractAuthSecurityConfig() { "/login", "/logout", "/oauth2/**", - "/static/**" + "/static/**", + "/metrics" }; } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/GraphsController.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/GraphsController.java new file mode 100644 index 00000000000..b4bbcd92710 --- /dev/null +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/GraphsController.java @@ -0,0 +1,79 @@ +package com.provectus.kafka.ui.controller; + +import com.provectus.kafka.ui.api.GraphsApi; +import com.provectus.kafka.ui.model.GraphDataRequestDTO; +import com.provectus.kafka.ui.model.GraphDescriptionDTO; +import com.provectus.kafka.ui.model.GraphDescriptionsDTO; +import com.provectus.kafka.ui.model.GraphParameterDTO; +import com.provectus.kafka.ui.model.PrometheusApiQueryResponseDTO; +import com.provectus.kafka.ui.model.rbac.AccessContext; +import com.provectus.kafka.ui.service.graphs.GraphDescription; +import com.provectus.kafka.ui.service.graphs.GraphsService; +import java.time.Duration; +import java.time.OffsetDateTime; +import java.util.Optional; +import lombok.RequiredArgsConstructor; +import org.mapstruct.Mapper; +import org.mapstruct.factory.Mappers; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.server.ServerWebExchange; +import prometheus.query.model.QueryResponse; +import reactor.core.publisher.Mono; + +@RestController +@RequiredArgsConstructor +public class GraphsController extends AbstractController implements GraphsApi { + + private static final PrometheusApiMapper MAPPER = Mappers.getMapper(PrometheusApiMapper.class); + + @Mapper + interface PrometheusApiMapper { + PrometheusApiQueryResponseDTO fromClientResponse(QueryResponse resp); + } + + private final GraphsService graphsService; + + @Override + public Mono> getGraphData(String clusterName, + Mono graphDataRequestDto, + ServerWebExchange exchange) { + var context = AccessContext.builder() + .cluster(clusterName) + .operationName("getGraphData") + .build(); + + return accessControlService.validateAccess(context) + .then( + graphDataRequestDto.flatMap(req -> + graphsService.getGraphData( + getCluster(clusterName), + req.getId(), + Optional.ofNullable(req.getFrom()).map(OffsetDateTime::toInstant).orElse(null), + Optional.ofNullable(req.getTo()).map(OffsetDateTime::toInstant).orElse(null), + req.getParameters() + ).map(MAPPER::fromClientResponse)) + .map(ResponseEntity::ok) + ).doOnEach(sig -> auditService.audit(context, sig)); + } + + @Override + public Mono> getGraphsList(String clusterName, + ServerWebExchange exchange) { + var context = AccessContext.builder() + .cluster(clusterName) + .operationName("getGraphsList") + .build(); + + var graphs = graphsService.getGraphs(getCluster(clusterName)); + return accessControlService.validateAccess(context).then( + Mono.just(ResponseEntity.ok(new GraphDescriptionsDTO().graphs(graphs.map(this::map).toList())))); + } + + private GraphDescriptionDTO map(GraphDescription graph) { + return new GraphDescriptionDTO(graph.id()) + .defaultPeriod(Optional.ofNullable(graph.defaultInterval()).map(Duration::toString).orElse(null)) + .type(graph.isRange() ? GraphDescriptionDTO.TypeEnum.RANGE : GraphDescriptionDTO.TypeEnum.INSTANT) + .parameters(graph.params().stream().map(GraphParameterDTO::new).toList()); + } +} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/PrometheusExposeController.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/PrometheusExposeController.java new file mode 100644 index 00000000000..1e964909a3f --- /dev/null +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/PrometheusExposeController.java @@ -0,0 +1,32 @@ +package com.provectus.kafka.ui.controller; + +import com.provectus.kafka.ui.api.PrometheusExposeApi; +import com.provectus.kafka.ui.model.KafkaCluster; +import com.provectus.kafka.ui.service.StatisticsCache; +import com.provectus.kafka.ui.service.metrics.prometheus.PrometheusExpose; +import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.server.ServerWebExchange; +import reactor.core.publisher.Mono; + +@RestController +@RequiredArgsConstructor +public class PrometheusExposeController extends AbstractController implements PrometheusExposeApi { + + private final StatisticsCache statisticsCache; + + @Override + public Mono> getAllMetrics(ServerWebExchange exchange) { + return Mono.just( + PrometheusExpose.exposeAllMetrics( + clustersStorage.getKafkaClusters() + .stream() + .filter(KafkaCluster::isExposeMetricsViaPrometheusEndpoint) + .collect(Collectors.toMap(KafkaCluster::getName, c -> statisticsCache.get(c).getMetrics())) + ) + ); + } + +} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/mapper/ClusterMapper.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/mapper/ClusterMapper.java index a122a269a4e..7b337f20da7 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/mapper/ClusterMapper.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/mapper/ClusterMapper.java @@ -1,9 +1,12 @@ package com.provectus.kafka.ui.mapper; +import static io.prometheus.client.Collector.MetricFamilySamples; +import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toMap; + import com.provectus.kafka.ui.config.ClustersProperties; import com.provectus.kafka.ui.model.BrokerConfigDTO; import com.provectus.kafka.ui.model.BrokerDTO; -import com.provectus.kafka.ui.model.BrokerDiskUsageDTO; import com.provectus.kafka.ui.model.BrokerMetricsDTO; import com.provectus.kafka.ui.model.ClusterDTO; import com.provectus.kafka.ui.model.ClusterFeature; @@ -14,7 +17,6 @@ import com.provectus.kafka.ui.model.ConnectDTO; import com.provectus.kafka.ui.model.InternalBroker; import com.provectus.kafka.ui.model.InternalBrokerConfig; -import com.provectus.kafka.ui.model.InternalBrokerDiskUsage; import com.provectus.kafka.ui.model.InternalClusterState; import com.provectus.kafka.ui.model.InternalPartition; import com.provectus.kafka.ui.model.InternalReplica; @@ -30,10 +32,13 @@ import com.provectus.kafka.ui.model.TopicConfigDTO; import com.provectus.kafka.ui.model.TopicDTO; import com.provectus.kafka.ui.model.TopicDetailsDTO; -import com.provectus.kafka.ui.service.metrics.RawMetric; +import com.provectus.kafka.ui.service.metrics.SummarizedMetrics; +import java.math.BigDecimal; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.Stream; import org.apache.kafka.clients.admin.ConfigEntry; import org.apache.kafka.common.acl.AccessControlEntry; import org.apache.kafka.common.acl.AclBinding; @@ -52,21 +57,28 @@ public interface ClusterMapper { ClusterStatsDTO toClusterStats(InternalClusterState clusterState); + @Deprecated default ClusterMetricsDTO toClusterMetrics(Metrics metrics) { return new ClusterMetricsDTO() - .items(metrics.getSummarizedMetrics().map(this::convert).collect(Collectors.toList())); + .items(convert(new SummarizedMetrics(metrics).asStream()).toList()); } - private MetricDTO convert(RawMetric rawMetric) { - return new MetricDTO() - .name(rawMetric.name()) - .labels(rawMetric.labels()) - .value(rawMetric.value()); + private Stream convert(Stream metrics) { + return metrics + .flatMap(m -> m.samples.stream()) + .map(s -> + new MetricDTO() + .name(s.name) + .labels(IntStream.range(0, s.labelNames.size()) + .boxed() + //collecting to map, keeping order + .collect(toMap(s.labelNames::get, s.labelValues::get, (m1, m2) -> null, LinkedHashMap::new))) + .value(BigDecimal.valueOf(s.value)) + ); } - default BrokerMetricsDTO toBrokerMetrics(List metrics) { - return new BrokerMetricsDTO() - .metrics(metrics.stream().map(this::convert).collect(Collectors.toList())); + default BrokerMetricsDTO toBrokerMetrics(List metrics) { + return new BrokerMetricsDTO().metrics(convert(metrics.stream()).toList()); } @Mapping(target = "isSensitive", source = "sensitive") @@ -107,15 +119,7 @@ default ConfigSynonymDTO toConfigSynonym(ConfigEntry.ConfigSynonym config) { List toFeaturesEnum(List features); default List map(Map map) { - return map.values().stream().map(this::toPartition).collect(Collectors.toList()); - } - - default BrokerDiskUsageDTO map(Integer id, InternalBrokerDiskUsage internalBrokerDiskUsage) { - final BrokerDiskUsageDTO brokerDiskUsage = new BrokerDiskUsageDTO(); - brokerDiskUsage.setBrokerId(id); - brokerDiskUsage.segmentCount((int) internalBrokerDiskUsage.getSegmentCount()); - brokerDiskUsage.segmentSize(internalBrokerDiskUsage.getSegmentSize()); - return brokerDiskUsage; + return map.values().stream().map(this::toPartition).collect(toList()); } static KafkaAclDTO.OperationEnum mapAclOperation(AclOperation operation) { diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/mapper/DescribeLogDirsMapper.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/mapper/DescribeLogDirsMapper.java index 3d84aa3ad90..bb37768dcc0 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/mapper/DescribeLogDirsMapper.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/mapper/DescribeLogDirsMapper.java @@ -7,6 +7,8 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import org.apache.kafka.clients.admin.LogDirDescription; +import org.apache.kafka.clients.admin.ReplicaInfo; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.requests.DescribeLogDirsResponse; @@ -16,7 +18,7 @@ public class DescribeLogDirsMapper { public List toBrokerLogDirsList( - Map> logDirsInfo) { + Map> logDirsInfo) { return logDirsInfo.entrySet().stream().map( mapEntry -> mapEntry.getValue().entrySet().stream() @@ -26,13 +28,13 @@ public List toBrokerLogDirsList( } private BrokersLogdirsDTO toBrokerLogDirs(Integer broker, String dirName, - DescribeLogDirsResponse.LogDirInfo logDirInfo) { + LogDirDescription logDirInfo) { BrokersLogdirsDTO result = new BrokersLogdirsDTO(); result.setName(dirName); - if (logDirInfo.error != null && logDirInfo.error != Errors.NONE) { - result.setError(logDirInfo.error.message()); + if (logDirInfo.error() != null) { + result.setError(logDirInfo.error().getMessage()); } - var topics = logDirInfo.replicaInfos.entrySet().stream() + var topics = logDirInfo.replicaInfos().entrySet().stream() .collect(Collectors.groupingBy(e -> e.getKey().topic())).entrySet().stream() .map(e -> toTopicLogDirs(broker, e.getKey(), e.getValue())) .collect(Collectors.toList()); @@ -41,8 +43,7 @@ private BrokersLogdirsDTO toBrokerLogDirs(Integer broker, String dirName, } private BrokerTopicLogdirsDTO toTopicLogDirs(Integer broker, String name, - List> partitions) { + List> partitions) { BrokerTopicLogdirsDTO topic = new BrokerTopicLogdirsDTO(); topic.setName(name); topic.setPartitions( @@ -54,13 +55,12 @@ private BrokerTopicLogdirsDTO toTopicLogDirs(Integer broker, String name, } private BrokerTopicPartitionLogdirDTO topicPartitionLogDir(Integer broker, Integer partition, - DescribeLogDirsResponse.ReplicaInfo - replicaInfo) { + ReplicaInfo replicaInfo) { BrokerTopicPartitionLogdirDTO logDir = new BrokerTopicPartitionLogdirDTO(); logDir.setBroker(broker); logDir.setPartition(partition); - logDir.setSize(replicaInfo.size); - logDir.setOffsetLag(replicaInfo.offsetLag); + logDir.setSize(replicaInfo.size()); + logDir.setOffsetLag(replicaInfo.offsetLag()); return logDir; } } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalBroker.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalBroker.java index 4a0d1ba0dd1..1a2c96d8c1c 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalBroker.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalBroker.java @@ -21,12 +21,12 @@ public class InternalBroker { public InternalBroker(Node node, PartitionDistributionStats partitionDistribution, - Statistics statistics) { + Metrics metrics) { this.id = node.id(); this.host = node.host(); this.port = node.port(); - this.bytesInPerSec = statistics.getMetrics().getBrokerBytesInPerSec().get(node.id()); - this.bytesOutPerSec = statistics.getMetrics().getBrokerBytesOutPerSec().get(node.id()); + this.bytesInPerSec = metrics.getIoRates().brokerBytesInPerSec().get(node.id()); + this.bytesOutPerSec = metrics.getIoRates().brokerBytesOutPerSec().get(node.id()); this.partitionsLeader = partitionDistribution.getPartitionLeaders().get(node); this.partitions = partitionDistribution.getPartitionsCount().get(node); this.inSyncPartitions = partitionDistribution.getInSyncPartitions().get(node); diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalBrokerDiskUsage.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalBrokerDiskUsage.java deleted file mode 100644 index 104051dc9eb..00000000000 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalBrokerDiskUsage.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.provectus.kafka.ui.model; - -import lombok.Builder; -import lombok.Data; - -@Data -@Builder(toBuilder = true) -public class InternalBrokerDiskUsage { - private final long segmentCount; - private final long segmentSize; -} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalClusterMetrics.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalClusterMetrics.java deleted file mode 100644 index 17aa8e51312..00000000000 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalClusterMetrics.java +++ /dev/null @@ -1,55 +0,0 @@ -package com.provectus.kafka.ui.model; - -import java.math.BigDecimal; -import java.util.List; -import java.util.Map; -import javax.annotation.Nullable; -import lombok.Builder; -import lombok.Data; - - -@Data -@Builder(toBuilder = true) -public class InternalClusterMetrics { - - public static InternalClusterMetrics empty() { - return InternalClusterMetrics.builder() - .brokers(List.of()) - .topics(Map.of()) - .status(ServerStatusDTO.OFFLINE) - .internalBrokerMetrics(Map.of()) - .metrics(List.of()) - .version("unknown") - .build(); - } - - private final String version; - - private final ServerStatusDTO status; - private final Throwable lastKafkaException; - - private final int brokerCount; - private final int activeControllers; - private final List brokers; - - private final int topicCount; - private final Map topics; - - // partitions stats - private final int underReplicatedPartitionCount; - private final int onlinePartitionCount; - private final int offlinePartitionCount; - private final int inSyncReplicasCount; - private final int outOfSyncReplicasCount; - - // log dir stats - @Nullable // will be null if log dir collection disabled - private final Map internalBrokerDiskUsage; - - // metrics from metrics collector - private final BigDecimal bytesInPerSec; - private final BigDecimal bytesOutPerSec; - private final Map internalBrokerMetrics; - private final List metrics; - -} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalClusterState.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalClusterState.java index 28e9a7413a3..201faa1f89a 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalClusterState.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalClusterState.java @@ -36,39 +36,42 @@ public InternalClusterState(KafkaCluster cluster, Statistics statistics) { .message(e.getMessage()) .stackTrace(Throwables.getStackTraceAsString(e))) .orElse(null); - topicCount = statistics.getTopicDescriptions().size(); + topicCount = (int) statistics.topicDescriptions().count(); brokerCount = statistics.getClusterDescription().getNodes().size(); activeControllers = Optional.ofNullable(statistics.getClusterDescription().getController()) .map(Node::id) .orElse(null); version = statistics.getVersion(); - if (statistics.getLogDirInfo() != null) { - diskUsage = statistics.getLogDirInfo().getBrokerStats().entrySet().stream() - .map(e -> new BrokerDiskUsageDTO() - .brokerId(e.getKey()) - .segmentSize(e.getValue().getSegmentSize()) - .segmentCount(e.getValue().getSegmentsCount())) - .collect(Collectors.toList()); - } + diskUsage = statistics.getClusterState().getNodesStates().values().stream() + .filter(n -> n.segmentStats() != null) + .map(n -> new BrokerDiskUsageDTO() + .brokerId(n.id()) + .segmentSize(n.segmentStats().getSegmentSize()) + .segmentCount(n.segmentStats().getSegmentsCount())) + .collect(Collectors.toList()); features = statistics.getFeatures(); bytesInPerSec = statistics .getMetrics() - .getBrokerBytesInPerSec() - .values().stream() + .getIoRates() + .brokerBytesInPerSec() + .values() + .stream() .reduce(BigDecimal::add) .orElse(null); bytesOutPerSec = statistics .getMetrics() - .getBrokerBytesOutPerSec() - .values().stream() + .getIoRates() + .brokerBytesOutPerSec() + .values() + .stream() .reduce(BigDecimal::add) .orElse(null); - var partitionsStats = new PartitionsStats(statistics.getTopicDescriptions().values()); + var partitionsStats = new PartitionsStats(statistics.topicDescriptions().toList()); onlinePartitionCount = partitionsStats.getOnlinePartitionCount(); offlinePartitionCount = partitionsStats.getOfflinePartitionCount(); inSyncReplicasCount = partitionsStats.getInSyncReplicasCount(); diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalLogDirStats.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalLogDirStats.java index 34ec3d59e3e..fa9b0f16169 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalLogDirStats.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalLogDirStats.java @@ -3,14 +3,17 @@ import static java.util.stream.Collectors.collectingAndThen; import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.summarizingLong; -import static java.util.stream.Collectors.toList; +import jakarta.annotation.Nullable; +import java.util.HashMap; import java.util.List; import java.util.LongSummaryStatistics; import java.util.Map; +import java.util.concurrent.atomic.LongAdder; +import lombok.RequiredArgsConstructor; import lombok.Value; +import org.apache.kafka.clients.admin.LogDirDescription; import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.requests.DescribeLogDirsResponse; import reactor.util.function.Tuple2; import reactor.util.function.Tuple3; import reactor.util.function.Tuples; @@ -19,32 +22,39 @@ public class InternalLogDirStats { @Value + @RequiredArgsConstructor public static class SegmentStats { - long segmentSize; - int segmentsCount; + Long segmentSize; + Integer segmentsCount; - public SegmentStats(LongSummaryStatistics s) { - segmentSize = s.getSum(); - segmentsCount = (int) s.getCount(); + private SegmentStats(LongSummaryStatistics s) { + this(s.getSum(), (int) s.getCount()); } } + public record LogDirSpaceStats(@Nullable Long totalBytes, + @Nullable Long usableBytes, + Map totalPerDir, + Map usablePerDir) { + } + Map partitionsStats; Map topicStats; Map brokerStats; + Map brokerDirsStats; public static InternalLogDirStats empty() { return new InternalLogDirStats(Map.of()); } - public InternalLogDirStats(Map> log) { + public InternalLogDirStats(Map> logsInfo) { final List> topicPartitions = - log.entrySet().stream().flatMap(b -> + logsInfo.entrySet().stream().flatMap(b -> b.getValue().entrySet().stream().flatMap(topicMap -> - topicMap.getValue().replicaInfos.entrySet().stream() - .map(e -> Tuples.of(b.getKey(), e.getKey(), e.getValue().size)) + topicMap.getValue().replicaInfos().entrySet().stream() + .map(e -> Tuples.of(b.getKey(), e.getKey(), e.getValue().size())) ) - ).collect(toList()); + ).toList(); partitionsStats = topicPartitions.stream().collect( groupingBy( @@ -64,5 +74,34 @@ public InternalLogDirStats(Map calculateSpaceStats( + Map> logsInfo) { + + var stats = new HashMap(); + logsInfo.forEach((brokerId, logDirStats) -> { + Map totalBytes = new HashMap<>(); + Map usableBytes = new HashMap<>(); + logDirStats.forEach((logDir, descr) -> { + if (descr.error() == null) { + return; + } + descr.totalBytes().ifPresent(b -> totalBytes.merge(logDir, b, Long::sum)); + descr.usableBytes().ifPresent(b -> usableBytes.merge(logDir, b, Long::sum)); + }); + stats.put( + brokerId, + new LogDirSpaceStats( + totalBytes.isEmpty() ? null : totalBytes.values().stream().mapToLong(i -> i).sum(), + usableBytes.isEmpty() ? null : usableBytes.values().stream().mapToLong(i -> i).sum(), + totalBytes, + usableBytes + ) + ); + }); + return stats; } } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalPartitionsOffsets.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalPartitionsOffsets.java index 9fb54a300ed..742320e53bc 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalPartitionsOffsets.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalPartitionsOffsets.java @@ -4,6 +4,7 @@ import com.google.common.collect.Table; import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; import lombok.Value; import org.apache.kafka.common.TopicPartition; @@ -30,4 +31,11 @@ public Optional get(String topic, int partition) { return Optional.ofNullable(offsets.get(topic, partition)); } + public Map topicOffsets(String topic, boolean earliest) { + return offsets.row(topic) + .entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> earliest ? e.getValue().earliest : e.getValue().getLatest())); + } + } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalSegmentSizeDto.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalSegmentSizeDto.java deleted file mode 100644 index a34db011321..00000000000 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalSegmentSizeDto.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.provectus.kafka.ui.model; - -import java.util.Map; -import lombok.Builder; -import lombok.Data; - -@Data -@Builder(toBuilder = true) -public class InternalSegmentSizeDto { - - private final Map internalTopicWithSegmentSize; - private final InternalClusterMetrics clusterMetricsWithSegmentSize; -} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalTopic.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalTopic.java index 43a6012d215..5a3aa149d65 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalTopic.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalTopic.java @@ -1,23 +1,22 @@ package com.provectus.kafka.ui.model; -import com.provectus.kafka.ui.config.ClustersProperties; +import static com.provectus.kafka.ui.model.InternalLogDirStats.SegmentStats; + import java.math.BigDecimal; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; import javax.annotation.Nullable; import lombok.Builder; import lombok.Data; import org.apache.kafka.clients.admin.ConfigEntry; import org.apache.kafka.clients.admin.TopicDescription; -import org.apache.kafka.common.TopicPartition; @Data @Builder(toBuilder = true) public class InternalTopic { - ClustersProperties clustersProperties; - // from TopicDescription private final String name; private final boolean internal; @@ -44,7 +43,8 @@ public static InternalTopic from(TopicDescription topicDescription, List configs, InternalPartitionsOffsets partitionsOffsets, Metrics metrics, - InternalLogDirStats logDirInfo, + @Nullable SegmentStats segmentStats, + @Nullable Map partitionsSegmentStats, @Nullable String internalTopicPrefix) { var topic = InternalTopic.builder(); @@ -81,13 +81,12 @@ public static InternalTopic from(TopicDescription topicDescription, partitionDto.offsetMax(offsets.getLatest()); }); - var segmentStats = - logDirInfo.getPartitionsStats().get( - new TopicPartition(topicDescription.name(), partition.partition())); - if (segmentStats != null) { - partitionDto.segmentCount(segmentStats.getSegmentsCount()); - partitionDto.segmentSize(segmentStats.getSegmentSize()); - } + Optional.ofNullable(partitionsSegmentStats) + .flatMap(s -> Optional.ofNullable(s.get(partition.partition()))) + .ifPresent(stats -> { + partitionDto.segmentCount(stats.getSegmentsCount()); + partitionDto.segmentSize(stats.getSegmentSize()); + }); return partitionDto.build(); }) @@ -108,14 +107,14 @@ public static InternalTopic from(TopicDescription topicDescription, : topicDescription.partitions().get(0).replicas().size() ); - var segmentStats = logDirInfo.getTopicStats().get(topicDescription.name()); - if (segmentStats != null) { - topic.segmentCount(segmentStats.getSegmentsCount()); - topic.segmentSize(segmentStats.getSegmentSize()); - } + Optional.ofNullable(segmentStats) + .ifPresent(stats -> { + topic.segmentCount(stats.getSegmentsCount()); + topic.segmentSize(stats.getSegmentSize()); + }); - topic.bytesInPerSec(metrics.getTopicBytesInPerSec().get(topicDescription.name())); - topic.bytesOutPerSec(metrics.getTopicBytesOutPerSec().get(topicDescription.name())); + topic.bytesInPerSec(metrics.getIoRates().topicBytesInPerSec().get(topicDescription.name())); + topic.bytesOutPerSec(metrics.getIoRates().topicBytesOutPerSec().get(topicDescription.name())); topic.topicConfigs( configs.stream().map(InternalTopicConfig::from).collect(Collectors.toList())); diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/KafkaCluster.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/KafkaCluster.java index 1e2903dbcc9..fc797e3811a 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/KafkaCluster.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/KafkaCluster.java @@ -5,6 +5,7 @@ import com.provectus.kafka.ui.emitter.PollingSettings; import com.provectus.kafka.ui.service.ksql.KsqlApiClient; import com.provectus.kafka.ui.service.masking.DataMasking; +import com.provectus.kafka.ui.service.metrics.scrape.MetricsScrapping; import com.provectus.kafka.ui.sr.api.KafkaSrClientApi; import com.provectus.kafka.ui.util.ReactiveFailover; import java.util.Map; @@ -13,6 +14,7 @@ import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; +import prometheus.query.api.PrometheusClientApi; @Data @Builder(toBuilder = true) @@ -25,10 +27,12 @@ public class KafkaCluster { private final String bootstrapServers; private final Properties properties; private final boolean readOnly; - private final MetricsConfig metricsConfig; + private final boolean exposeMetricsViaPrometheusEndpoint; private final DataMasking masking; private final PollingSettings pollingSettings; private final ReactiveFailover schemaRegistryClient; private final Map> connectsClients; private final ReactiveFailover ksqlClient; + private final MetricsScrapping metricsScrapping; + private final ReactiveFailover prometheusStorageClient; } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/Metrics.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/Metrics.java index 02bfe6dea13..c0f9737da0f 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/Metrics.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/Metrics.java @@ -1,13 +1,11 @@ package com.provectus.kafka.ui.model; -import static java.util.stream.Collectors.toMap; +import static io.prometheus.client.Collector.MetricFamilySamples; -import com.provectus.kafka.ui.service.metrics.RawMetric; +import com.provectus.kafka.ui.service.metrics.scrape.inferred.InferredMetrics; import java.math.BigDecimal; -import java.util.Collection; import java.util.List; import java.util.Map; -import java.util.stream.Stream; import lombok.Builder; import lombok.Value; @@ -16,28 +14,32 @@ @Value public class Metrics { - Map brokerBytesInPerSec; - Map brokerBytesOutPerSec; - Map topicBytesInPerSec; - Map topicBytesOutPerSec; - Map> perBrokerMetrics; + IoRates ioRates; + InferredMetrics inferredMetrics; + Map> perBrokerScrapedMetrics; public static Metrics empty() { return Metrics.builder() - .brokerBytesInPerSec(Map.of()) - .brokerBytesOutPerSec(Map.of()) - .topicBytesInPerSec(Map.of()) - .topicBytesOutPerSec(Map.of()) - .perBrokerMetrics(Map.of()) + .ioRates(IoRates.empty()) + .perBrokerScrapedMetrics(Map.of()) + .inferredMetrics(InferredMetrics.empty()) .build(); } - public Stream getSummarizedMetrics() { - return perBrokerMetrics.values().stream() - .flatMap(Collection::stream) - .collect(toMap(RawMetric::identityKey, m -> m, (m1, m2) -> m1.copyWithValue(m1.value().add(m2.value())))) - .values() - .stream(); + @Builder + public record IoRates(Map brokerBytesInPerSec, + Map brokerBytesOutPerSec, + Map topicBytesInPerSec, + Map topicBytesOutPerSec) { + + static IoRates empty() { + return IoRates.builder() + .brokerBytesOutPerSec(Map.of()) + .brokerBytesInPerSec(Map.of()) + .topicBytesOutPerSec(Map.of()) + .topicBytesInPerSec(Map.of()) + .build(); + } } } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/MetricsConfig.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/MetricsConfig.java deleted file mode 100644 index d3551443437..00000000000 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/MetricsConfig.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.provectus.kafka.ui.model; - -import lombok.AccessLevel; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; - -@Data -@Builder(toBuilder = true) -@AllArgsConstructor(access = AccessLevel.PRIVATE) -public class MetricsConfig { - public static final String JMX_METRICS_TYPE = "JMX"; - public static final String PROMETHEUS_METRICS_TYPE = "PROMETHEUS"; - - private final String type; - private final Integer port; - private final boolean ssl; - private final String username; - private final String password; - private final String keystoreLocation; - private final String keystorePassword; -} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/MetricsScrapeProperties.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/MetricsScrapeProperties.java new file mode 100644 index 00000000000..2e64b0253db --- /dev/null +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/MetricsScrapeProperties.java @@ -0,0 +1,46 @@ +package com.provectus.kafka.ui.model; + +import static com.provectus.kafka.ui.config.ClustersProperties.KeystoreConfig; +import static com.provectus.kafka.ui.config.ClustersProperties.TruststoreConfig; + +import com.provectus.kafka.ui.config.ClustersProperties; +import jakarta.annotation.Nullable; +import java.util.Objects; +import java.util.Optional; +import lombok.Builder; +import lombok.Value; + +@Value +@Builder +public class MetricsScrapeProperties { + public static final String JMX_METRICS_TYPE = "JMX"; + public static final String PROMETHEUS_METRICS_TYPE = "PROMETHEUS"; + + Integer port; + boolean ssl; + String username; + String password; + + @Nullable + KeystoreConfig keystoreConfig; + + @Nullable + TruststoreConfig truststoreConfig; + + public static MetricsScrapeProperties create(ClustersProperties.Cluster cluster) { + var metrics = Objects.requireNonNull(cluster.getMetrics()); + return MetricsScrapeProperties.builder() + .port(metrics.getPort()) + .ssl(Optional.ofNullable(metrics.getSsl()).orElse(false)) + .username(metrics.getUsername()) + .password(metrics.getPassword()) + .truststoreConfig(cluster.getSsl()) + .keystoreConfig( + metrics.getKeystoreLocation() != null + ? new KeystoreConfig(metrics.getKeystoreLocation(), metrics.getKeystorePassword()) + : null + ) + .build(); + } + +} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/PartitionDistributionStats.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/PartitionDistributionStats.java index 46efc670008..3ba498618be 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/PartitionDistributionStats.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/PartitionDistributionStats.java @@ -1,14 +1,17 @@ package com.provectus.kafka.ui.model; +import com.provectus.kafka.ui.service.metrics.scrape.ScrapedClusterState; import java.math.BigDecimal; import java.math.RoundingMode; import java.util.HashMap; +import java.util.List; import java.util.Map; import javax.annotation.Nullable; import lombok.AccessLevel; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.mutable.MutableInt; import org.apache.kafka.clients.admin.TopicDescription; import org.apache.kafka.common.Node; import org.apache.kafka.common.TopicPartitionInfo; @@ -29,15 +32,19 @@ public class PartitionDistributionStats { private final boolean skewCanBeCalculated; public static PartitionDistributionStats create(Statistics stats) { - return create(stats, MIN_PARTITIONS_FOR_SKEW_CALCULATION); + return create( + stats.topicDescriptions().toList(), + MIN_PARTITIONS_FOR_SKEW_CALCULATION + ); } - static PartitionDistributionStats create(Statistics stats, int minPartitionsForSkewCalculation) { + static PartitionDistributionStats create(List topicDescriptions, + int minPartitionsForSkewCalculation) { var partitionLeaders = new HashMap(); var partitionsReplicated = new HashMap(); var isr = new HashMap(); int partitionsCnt = 0; - for (TopicDescription td : stats.getTopicDescriptions().values()) { + for (TopicDescription td : topicDescriptions) { for (TopicPartitionInfo tp : td.partitions()) { partitionsCnt++; tp.replicas().forEach(r -> incr(partitionsReplicated, r)); diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/Statistics.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/Statistics.java index e70547f1437..998ab2891e8 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/Statistics.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/Statistics.java @@ -1,9 +1,11 @@ package com.provectus.kafka.ui.model; import com.provectus.kafka.ui.service.ReactiveAdminClient; +import com.provectus.kafka.ui.service.metrics.scrape.ScrapedClusterState; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Stream; import lombok.Builder; import lombok.Value; import org.apache.kafka.clients.admin.ConfigEntry; @@ -18,9 +20,7 @@ public class Statistics { List features; ReactiveAdminClient.ClusterDescription clusterDescription; Metrics metrics; - InternalLogDirStats logDirInfo; - Map topicDescriptions; - Map> topicConfigs; + ScrapedClusterState clusterState; public static Statistics empty() { return builder() @@ -30,9 +30,12 @@ public static Statistics empty() { .clusterDescription( new ReactiveAdminClient.ClusterDescription(null, null, List.of(), Set.of())) .metrics(Metrics.empty()) - .logDirInfo(InternalLogDirStats.empty()) - .topicDescriptions(Map.of()) - .topicConfigs(Map.of()) + .clusterState(ScrapedClusterState.empty()) .build(); } + + public Stream topicDescriptions() { + return clusterState.getTopicStates().values().stream().map(ScrapedClusterState.TopicState::description); + } + } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/BrokerService.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/BrokerService.java index ff86d2be0f7..b678b7e95fa 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/BrokerService.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/BrokerService.java @@ -1,5 +1,7 @@ package com.provectus.kafka.ui.service; +import static io.prometheus.client.Collector.MetricFamilySamples; + import com.provectus.kafka.ui.exception.InvalidRequestApiException; import com.provectus.kafka.ui.exception.LogDirNotFoundApiException; import com.provectus.kafka.ui.exception.NotFoundException; @@ -11,7 +13,6 @@ import com.provectus.kafka.ui.model.InternalBrokerConfig; import com.provectus.kafka.ui.model.KafkaCluster; import com.provectus.kafka.ui.model.PartitionDistributionStats; -import com.provectus.kafka.ui.service.metrics.RawMetric; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -21,13 +22,13 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.admin.ConfigEntry; +import org.apache.kafka.clients.admin.LogDirDescription; import org.apache.kafka.common.Node; import org.apache.kafka.common.TopicPartitionReplica; import org.apache.kafka.common.errors.InvalidRequestException; import org.apache.kafka.common.errors.LogDirNotFoundException; import org.apache.kafka.common.errors.TimeoutException; import org.apache.kafka.common.errors.UnknownTopicOrPartitionException; -import org.apache.kafka.common.requests.DescribeLogDirsResponse; import org.springframework.stereotype.Service; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -72,7 +73,7 @@ public Flux getBrokers(KafkaCluster cluster) { .get(cluster) .flatMap(ReactiveAdminClient::describeCluster) .map(description -> description.getNodes().stream() - .map(node -> new InternalBroker(node, partitionsDistribution, stats)) + .map(node -> new InternalBroker(node, partitionsDistribution, stats.getMetrics())) .collect(Collectors.toList())) .flatMapMany(Flux::fromIterable); } @@ -110,7 +111,7 @@ public Mono updateBrokerConfigByName(KafkaCluster cluster, .doOnError(e -> log.error("Unexpected error", e)); } - private Mono>> getClusterLogDirs( + private Mono>> getClusterLogDirs( KafkaCluster cluster, List reqBrokers) { return adminClientService.get(cluster) .flatMap(admin -> { @@ -139,8 +140,8 @@ public Flux getBrokerConfig(KafkaCluster cluster, Integer return getBrokersConfig(cluster, brokerId); } - public Mono> getBrokerMetrics(KafkaCluster cluster, Integer brokerId) { - return Mono.justOrEmpty(statisticsCache.get(cluster).getMetrics().getPerBrokerMetrics().get(brokerId)); + public Mono> getBrokerMetrics(KafkaCluster cluster, Integer brokerId) { + return Mono.justOrEmpty(statisticsCache.get(cluster).getMetrics().getPerBrokerScrapedMetrics().get(brokerId)); } } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/ClustersStatisticsScheduler.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/ClustersStatisticsScheduler.java index f7ac1239a04..80d4e7d1113 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/ClustersStatisticsScheduler.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/ClustersStatisticsScheduler.java @@ -22,9 +22,9 @@ public void updateStatistics() { .parallel() .runOn(Schedulers.parallel()) .flatMap(cluster -> { - log.debug("Start getting metrics for kafkaCluster: {}", cluster.getName()); + log.debug("Start collection statistics for cluster: {}", cluster.getName()); return statisticsService.updateCache(cluster) - .doOnSuccess(m -> log.debug("Metrics updated for cluster: {}", cluster.getName())); + .doOnSuccess(m -> log.debug("Statistics updated for cluster: {}", cluster.getName())); }) .then() .block(); diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/KafkaClusterFactory.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/KafkaClusterFactory.java index 964b25473d3..3d13d0c878a 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/KafkaClusterFactory.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/KafkaClusterFactory.java @@ -1,5 +1,12 @@ package com.provectus.kafka.ui.service; +import static com.provectus.kafka.ui.util.KafkaServicesValidation.validateClusterConnection; +import static com.provectus.kafka.ui.util.KafkaServicesValidation.validateConnect; +import static com.provectus.kafka.ui.util.KafkaServicesValidation.validateKsql; +import static com.provectus.kafka.ui.util.KafkaServicesValidation.validatePrometheusStore; +import static com.provectus.kafka.ui.util.KafkaServicesValidation.validateSchemaRegistry; +import static com.provectus.kafka.ui.util.KafkaServicesValidation.validateTruststore; + import com.provectus.kafka.ui.client.RetryingKafkaConnectClient; import com.provectus.kafka.ui.config.ClustersProperties; import com.provectus.kafka.ui.config.WebclientProperties; @@ -8,9 +15,10 @@ import com.provectus.kafka.ui.model.ApplicationPropertyValidationDTO; import com.provectus.kafka.ui.model.ClusterConfigValidationDTO; import com.provectus.kafka.ui.model.KafkaCluster; -import com.provectus.kafka.ui.model.MetricsConfig; import com.provectus.kafka.ui.service.ksql.KsqlApiClient; import com.provectus.kafka.ui.service.masking.DataMasking; +import com.provectus.kafka.ui.service.metrics.scrape.MetricsScrapping; +import com.provectus.kafka.ui.service.metrics.scrape.jmx.JmxMetricsRetriever; import com.provectus.kafka.ui.sr.ApiClient; import com.provectus.kafka.ui.sr.api.KafkaSrClientApi; import com.provectus.kafka.ui.util.KafkaServicesValidation; @@ -22,11 +30,12 @@ import java.util.Optional; import java.util.Properties; import java.util.stream.Stream; -import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; import org.springframework.util.unit.DataSize; import org.springframework.web.reactive.function.client.WebClient; +import prometheus.query.api.PrometheusClientApi; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.util.function.Tuple2; @@ -39,11 +48,13 @@ public class KafkaClusterFactory { private static final DataSize DEFAULT_WEBCLIENT_BUFFER = DataSize.parse("20MB"); private final DataSize webClientMaxBuffSize; + private final JmxMetricsRetriever jmxMetricsRetriever; - public KafkaClusterFactory(WebclientProperties webclientProperties) { + public KafkaClusterFactory(WebclientProperties webclientProperties, JmxMetricsRetriever jmxMetricsRetriever) { this.webClientMaxBuffSize = Optional.ofNullable(webclientProperties.getMaxInMemoryBufferSize()) .map(DataSize::parse) .orElse(DEFAULT_WEBCLIENT_BUFFER); + this.jmxMetricsRetriever = jmxMetricsRetriever; } public KafkaCluster create(ClustersProperties properties, @@ -54,8 +65,10 @@ public KafkaCluster create(ClustersProperties properties, builder.bootstrapServers(clusterProperties.getBootstrapServers()); builder.properties(convertProperties(clusterProperties.getProperties())); builder.readOnly(clusterProperties.isReadOnly()); + builder.exposeMetricsViaPrometheusEndpoint(exposeMetricsViaPrometheusEndpoint(clusterProperties)); builder.masking(DataMasking.create(clusterProperties.getMasking())); builder.pollingSettings(PollingSettings.create(clusterProperties, properties)); + builder.metricsScrapping(MetricsScrapping.create(clusterProperties, jmxMetricsRetriever)); if (schemaRegistryConfigured(clusterProperties)) { builder.schemaRegistryClient(schemaRegistryClient(clusterProperties)); @@ -66,8 +79,8 @@ public KafkaCluster create(ClustersProperties properties, if (ksqlConfigured(clusterProperties)) { builder.ksqlClient(ksqlClient(clusterProperties)); } - if (metricsConfigured(clusterProperties)) { - builder.metricsConfig(metricsConfigDataToMetricsConfig(clusterProperties.getMetrics())); + if (prometheusStorageConfigured(clusterProperties)) { + builder.prometheusStorageClient(prometheusStorageClient(clusterProperties)); } builder.originalProperties(clusterProperties); return builder.build(); @@ -75,7 +88,7 @@ public KafkaCluster create(ClustersProperties properties, public Mono validate(ClustersProperties.Cluster clusterProperties) { if (clusterProperties.getSsl() != null) { - Optional errMsg = KafkaServicesValidation.validateTruststore(clusterProperties.getSsl()); + Optional errMsg = validateTruststore(clusterProperties.getSsl()); if (errMsg.isPresent()) { return Mono.just(new ClusterConfigValidationDTO() .kafka(new ApplicationPropertyValidationDTO() @@ -85,40 +98,51 @@ public Mono validate(ClustersProperties.Cluster clus } return Mono.zip( - KafkaServicesValidation.validateClusterConnection( + validateClusterConnection( clusterProperties.getBootstrapServers(), convertProperties(clusterProperties.getProperties()), clusterProperties.getSsl() ), schemaRegistryConfigured(clusterProperties) - ? KafkaServicesValidation.validateSchemaRegistry( - () -> schemaRegistryClient(clusterProperties)).map(Optional::of) + ? validateSchemaRegistry(() -> schemaRegistryClient(clusterProperties)).map(Optional::of) : Mono.>just(Optional.empty()), ksqlConfigured(clusterProperties) - ? KafkaServicesValidation.validateKsql(() -> ksqlClient(clusterProperties)).map(Optional::of) + ? validateKsql(() -> ksqlClient(clusterProperties)).map(Optional::of) : Mono.>just(Optional.empty()), connectClientsConfigured(clusterProperties) ? Flux.fromIterable(clusterProperties.getKafkaConnect()) .flatMap(c -> - KafkaServicesValidation.validateConnect(() -> connectClient(clusterProperties, c)) + validateConnect(() -> connectClient(clusterProperties, c)) .map(r -> Tuples.of(c.getName(), r))) .collectMap(Tuple2::getT1, Tuple2::getT2) .map(Optional::of) : - Mono.>>just(Optional.empty()) + Mono.>>just(Optional.empty()), + + prometheusStorageConfigured(clusterProperties) + ? validatePrometheusStore(() -> prometheusStorageClient(clusterProperties)).map(Optional::of) + : Mono.>just(Optional.empty()) + ).map(tuple -> { var validation = new ClusterConfigValidationDTO(); validation.kafka(tuple.getT1()); tuple.getT2().ifPresent(validation::schemaRegistry); tuple.getT3().ifPresent(validation::ksqldb); tuple.getT4().ifPresent(validation::kafkaConnects); + tuple.getT5().ifPresent(validation::prometheusStorage); return validation; }); } + private boolean exposeMetricsViaPrometheusEndpoint(ClustersProperties.Cluster clusterProperties) { + return Optional.ofNullable(clusterProperties.getMetrics()) + .map(m -> m.getPrometheusExpose() == null || m.getPrometheusExpose()) + .orElse(true); + } + private Properties convertProperties(Map propertiesMap) { Properties properties = new Properties(); if (propertiesMap != null) { @@ -153,6 +177,28 @@ private ReactiveFailover connectClient(ClustersProperties ); } + private ReactiveFailover prometheusStorageClient(ClustersProperties.Cluster cluster) { + WebClient webClient = new WebClientConfigurator() + .configureSsl(cluster.getSsl(), null) + .configureBufferSize(webClientMaxBuffSize) + .build(); + return ReactiveFailover.create( + parseUrlList(cluster.getMetrics().getStore().getPrometheus().getUrl()), + url -> new PrometheusClientApi(new prometheus.query.ApiClient(webClient).setBasePath(url)), + ReactiveFailover.CONNECTION_REFUSED_EXCEPTION_FILTER, + "No live schemaRegistry instances available", + ReactiveFailover.DEFAULT_RETRY_GRACE_PERIOD_MS + ); + } + + private boolean prometheusStorageConfigured(ClustersProperties.Cluster cluster) { + return Optional.ofNullable(cluster.getMetrics()) + .flatMap(m -> Optional.ofNullable(m.getStore())) + .flatMap(s -> Optional.ofNullable(s.getPrometheus())) + .map(p -> StringUtils.hasText(p.getUrl())) + .orElse(false); + } + private boolean schemaRegistryConfigured(ClustersProperties.Cluster clusterProperties) { return clusterProperties.getSchemaRegistry() != null; } @@ -202,20 +248,4 @@ private boolean metricsConfigured(ClustersProperties.Cluster clusterProperties) return clusterProperties.getMetrics() != null; } - @Nullable - private MetricsConfig metricsConfigDataToMetricsConfig(ClustersProperties.MetricsConfigData metricsConfigData) { - if (metricsConfigData == null) { - return null; - } - MetricsConfig.MetricsConfigBuilder builder = MetricsConfig.builder(); - builder.type(metricsConfigData.getType()); - builder.port(metricsConfigData.getPort()); - builder.ssl(Optional.ofNullable(metricsConfigData.getSsl()).orElse(false)); - builder.username(metricsConfigData.getUsername()); - builder.password(metricsConfigData.getPassword()); - builder.keystoreLocation(metricsConfigData.getKeystoreLocation()); - builder.keystorePassword(metricsConfigData.getKeystorePassword()); - return builder.build(); - } - } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/MessagesService.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/MessagesService.java index 620bd840861..13719da8e56 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/MessagesService.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/MessagesService.java @@ -187,13 +187,18 @@ private Mono sendMessageImpl(KafkaCluster cluster, public static KafkaProducer createProducer(KafkaCluster cluster, Map additionalProps) { + return createProducer(cluster.getOriginalProperties(), additionalProps); + } + + public static KafkaProducer createProducer(ClustersProperties.Cluster cluster, + Map additionalProps) { Properties properties = new Properties(); - SslPropertiesUtil.addKafkaSslProperties(cluster.getOriginalProperties().getSsl(), properties); + SslPropertiesUtil.addKafkaSslProperties(cluster.getSsl(), properties); + properties.putAll(additionalProps); properties.putAll(cluster.getProperties()); properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, cluster.getBootstrapServers()); properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); - properties.putAll(additionalProps); return new KafkaProducer<>(properties); } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/ReactiveAdminClient.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/ReactiveAdminClient.java index 9de908efa7f..08b7bffc881 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/ReactiveAdminClient.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/ReactiveAdminClient.java @@ -51,6 +51,7 @@ import org.apache.kafka.clients.admin.ListConsumerGroupOffsetsSpec; import org.apache.kafka.clients.admin.ListOffsetsResult; import org.apache.kafka.clients.admin.ListTopicsOptions; +import org.apache.kafka.clients.admin.LogDirDescription; import org.apache.kafka.clients.admin.NewPartitionReassignment; import org.apache.kafka.clients.admin.NewPartitions; import org.apache.kafka.clients.admin.NewTopic; @@ -77,7 +78,6 @@ import org.apache.kafka.common.errors.TopicAuthorizationException; import org.apache.kafka.common.errors.UnknownTopicOrPartitionException; import org.apache.kafka.common.errors.UnsupportedVersionException; -import org.apache.kafka.common.requests.DescribeLogDirsResponse; import org.apache.kafka.common.resource.ResourcePatternFilter; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -378,15 +378,8 @@ static Mono> toMonoWithExceptionFilter(Map> v ); } - public Mono>> describeLogDirs() { - return describeCluster() - .map(d -> d.getNodes().stream().map(Node::id).collect(toList())) - .flatMap(this::describeLogDirs); - } - - public Mono>> describeLogDirs( - Collection brokerIds) { - return toMono(client.describeLogDirs(brokerIds).all()) + public Mono>> describeLogDirs(Collection brokerIds) { + return toMono(client.describeLogDirs(brokerIds).allDescriptions()) .onErrorResume(UnsupportedVersionException.class, th -> Mono.just(Map.of())) .onErrorResume(ClusterAuthorizationException.class, th -> Mono.just(Map.of())) .onErrorResume(th -> true, th -> { diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/StatisticsCache.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/StatisticsCache.java index 3acd64262b0..abfac1e43f6 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/StatisticsCache.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/StatisticsCache.java @@ -1,5 +1,6 @@ package com.provectus.kafka.ui.service; +import com.provectus.kafka.ui.model.InternalPartitionsOffsets; import com.provectus.kafka.ui.model.KafkaCluster; import com.provectus.kafka.ui.model.ServerStatusDTO; import com.provectus.kafka.ui.model.Statistics; @@ -28,38 +29,29 @@ public synchronized void replace(KafkaCluster c, Statistics stats) { public synchronized void update(KafkaCluster c, Map descriptions, - Map> configs) { - var metrics = get(c); - var updatedDescriptions = new HashMap<>(metrics.getTopicDescriptions()); - updatedDescriptions.putAll(descriptions); - var updatedConfigs = new HashMap<>(metrics.getTopicConfigs()); - updatedConfigs.putAll(configs); + Map> configs, + InternalPartitionsOffsets partitionsOffsets) { + var stats = get(c); replace( c, - metrics.toBuilder() - .topicDescriptions(updatedDescriptions) - .topicConfigs(updatedConfigs) + stats.toBuilder() + .clusterState(stats.getClusterState().updateTopics(descriptions, configs, partitionsOffsets)) .build() ); } public synchronized void onTopicDelete(KafkaCluster c, String topic) { - var metrics = get(c); - var updatedDescriptions = new HashMap<>(metrics.getTopicDescriptions()); - updatedDescriptions.remove(topic); - var updatedConfigs = new HashMap<>(metrics.getTopicConfigs()); - updatedConfigs.remove(topic); + var stats = get(c); replace( c, - metrics.toBuilder() - .topicDescriptions(updatedDescriptions) - .topicConfigs(updatedConfigs) + stats.toBuilder() + .clusterState(stats.getClusterState().topicDeleted(topic)) .build() ); } public Statistics get(KafkaCluster c) { - return Objects.requireNonNull(cache.get(c.getName()), "Unknown cluster metrics requested"); + return Objects.requireNonNull(cache.get(c.getName()), "Statistics for unknown cluster requested"); } } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/StatisticsService.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/StatisticsService.java index 19d946590c4..c1f7af021ef 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/StatisticsService.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/StatisticsService.java @@ -2,21 +2,16 @@ import static com.provectus.kafka.ui.service.ReactiveAdminClient.ClusterDescription; -import com.provectus.kafka.ui.model.ClusterFeature; import com.provectus.kafka.ui.model.InternalLogDirStats; import com.provectus.kafka.ui.model.KafkaCluster; import com.provectus.kafka.ui.model.Metrics; import com.provectus.kafka.ui.model.ServerStatusDTO; import com.provectus.kafka.ui.model.Statistics; -import com.provectus.kafka.ui.service.metrics.MetricsCollector; -import java.util.List; +import com.provectus.kafka.ui.service.metrics.scrape.ScrapedClusterState; import java.util.Map; import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.admin.ConfigEntry; -import org.apache.kafka.clients.admin.TopicDescription; -import org.apache.kafka.common.Node; import org.springframework.stereotype.Service; import reactor.core.publisher.Mono; @@ -25,7 +20,6 @@ @Slf4j public class StatisticsService { - private final MetricsCollector metricsCollector; private final AdminClientService adminClientService; private final FeatureService featureService; private final StatisticsCache cache; @@ -36,44 +30,38 @@ public Mono updateCache(KafkaCluster c) { private Mono getStatistics(KafkaCluster cluster) { return adminClientService.get(cluster).flatMap(ac -> - ac.describeCluster().flatMap(description -> - ac.updateInternalStats(description.getController()).then( - Mono.zip( - List.of( - metricsCollector.getBrokerMetrics(cluster, description.getNodes()), - getLogDirInfo(description, ac), - featureService.getAvailableFeatures(ac, cluster, description), - loadTopicConfigs(cluster), - describeTopics(cluster)), - results -> - Statistics.builder() - .status(ServerStatusDTO.ONLINE) - .clusterDescription(description) - .version(ac.getVersion()) - .metrics((Metrics) results[0]) - .logDirInfo((InternalLogDirStats) results[1]) - .features((List) results[2]) - .topicConfigs((Map>) results[3]) - .topicDescriptions((Map) results[4]) - .build() - )))) + ac.describeCluster() + .flatMap(description -> + ac.updateInternalStats(description.getController()) + .then( + Mono.zip( + featureService.getAvailableFeatures(ac, cluster, description), + loadClusterState(description, ac) + ).flatMap(featuresAndState -> + scrapeMetrics(cluster, featuresAndState.getT2(), description) + .map(metrics -> + Statistics.builder() + .status(ServerStatusDTO.ONLINE) + .clusterDescription(description) + .version(ac.getVersion()) + .metrics(metrics) + .features(featuresAndState.getT1()) + .clusterState(featuresAndState.getT2()) + .build()))))) .doOnError(e -> log.error("Failed to collect cluster {} info", cluster.getName(), e)) .onErrorResume( e -> Mono.just(Statistics.empty().toBuilder().lastKafkaException(e).build())); } - private Mono getLogDirInfo(ClusterDescription desc, ReactiveAdminClient ac) { - var brokerIds = desc.getNodes().stream().map(Node::id).collect(Collectors.toSet()); - return ac.describeLogDirs(brokerIds).map(InternalLogDirStats::new); + private Mono loadClusterState(ClusterDescription clusterDescription, ReactiveAdminClient ac) { + return ScrapedClusterState.scrape(clusterDescription, ac); } - private Mono> describeTopics(KafkaCluster c) { - return adminClientService.get(c).flatMap(ReactiveAdminClient::describeTopics); - } - - private Mono>> loadTopicConfigs(KafkaCluster c) { - return adminClientService.get(c).flatMap(ReactiveAdminClient::getTopicsConfig); + private Mono scrapeMetrics(KafkaCluster cluster, + ScrapedClusterState clusterState, + ClusterDescription clusterDescription) { + return cluster.getMetricsScrapping().scrape(clusterState, clusterDescription.getNodes()); } } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/TopicsService.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/TopicsService.java index 3ddcd6f82e0..1d46a648f6e 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/TopicsService.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/TopicsService.java @@ -1,5 +1,6 @@ package com.provectus.kafka.ui.service; +import static com.provectus.kafka.ui.service.metrics.scrape.ScrapedClusterState.TopicState; import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toMap; @@ -25,6 +26,7 @@ import com.provectus.kafka.ui.model.Statistics; import com.provectus.kafka.ui.model.TopicCreationDTO; import com.provectus.kafka.ui.model.TopicUpdateDTO; +import com.provectus.kafka.ui.service.metrics.scrape.ScrapedClusterState; import java.time.Duration; import java.util.Collection; import java.util.Collections; @@ -71,20 +73,19 @@ public Mono> loadTopics(KafkaCluster c, List topics) return adminClientService.get(c) .flatMap(ac -> ac.describeTopics(topics).zipWith(ac.getTopicsConfig(topics, false), - (descriptions, configs) -> { - statisticsCache.update(c, descriptions, configs); - return getPartitionOffsets(descriptions, ac).map(offsets -> { - var metrics = statisticsCache.get(c); - return createList( - topics, - descriptions, - configs, - offsets, - metrics.getMetrics(), - metrics.getLogDirInfo() - ); - }); - })).flatMap(Function.identity()); + (descriptions, configs) -> + getPartitionOffsets(descriptions, ac).map(offsets -> { + statisticsCache.update(c, descriptions, configs, offsets); + var stats = statisticsCache.get(c); + return createList( + topics, + descriptions, + configs, + offsets, + stats.getMetrics(), + stats.getClusterState() + ); + }))).flatMap(Function.identity()); } private Mono loadTopic(KafkaCluster c, String topicName) { @@ -95,8 +96,8 @@ private Mono loadTopic(KafkaCluster c, String topicName) { } /** - * After creation topic can be invisible via API for some time. - * To workaround this, we retyring topic loading until it becomes visible. + * After creation topic can be invisible via API for some time. + * To workaround this, we retyring topic loading until it becomes visible. */ private Mono loadTopicAfterCreation(KafkaCluster c, String topicName) { return loadTopic(c, topicName) @@ -122,7 +123,7 @@ private List createList(List orderedNames, Map> configs, InternalPartitionsOffsets partitionsOffsets, Metrics metrics, - InternalLogDirStats logDirInfo) { + ScrapedClusterState clusterState) { return orderedNames.stream() .filter(descriptions::containsKey) .map(t -> InternalTopic.from( @@ -130,7 +131,8 @@ private List createList(List orderedNames, configs.getOrDefault(t, List.of()), partitionsOffsets, metrics, - logDirInfo, + Optional.ofNullable(clusterState.getTopicStates().get(t)).map(s -> s.segmentStats()).orElse(null), + Optional.ofNullable(clusterState.getTopicStates().get(t)).map(s -> s.partitionsSegmentStats()).orElse(null), clustersProperties.getInternalTopicPrefix() )) .collect(toList()); @@ -225,7 +227,7 @@ private Mono updateTopic(KafkaCluster cluster, } public Mono updateTopic(KafkaCluster cl, String topicName, - Mono topicUpdate) { + Mono topicUpdate) { return topicUpdate .flatMap(t -> updateTopic(cl, topicName, t)); } @@ -444,17 +446,21 @@ public Mono cloneTopic( public Mono> getTopicsForPagination(KafkaCluster cluster) { Statistics stats = statisticsCache.get(cluster); - return filterExisting(cluster, stats.getTopicDescriptions().keySet()) + Map topicStates = stats.getClusterState().getTopicStates(); + return filterExisting(cluster, topicStates.keySet()) .map(lst -> lst.stream() .map(topicName -> InternalTopic.from( - stats.getTopicDescriptions().get(topicName), - stats.getTopicConfigs().getOrDefault(topicName, List.of()), + topicStates.get(topicName).description(), + topicStates.get(topicName).configs(), InternalPartitionsOffsets.empty(), stats.getMetrics(), - stats.getLogDirInfo(), + Optional.ofNullable(topicStates.get(topicName)) + .map(TopicState::segmentStats).orElse(null), + Optional.ofNullable(topicStates.get(topicName)) + .map(TopicState::partitionsSegmentStats).orElse(null), clustersProperties.getInternalTopicPrefix() - )) + )) .collect(toList()) ); } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/graphs/GraphDescription.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/graphs/GraphDescription.java new file mode 100644 index 00000000000..497f84b5761 --- /dev/null +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/graphs/GraphDescription.java @@ -0,0 +1,25 @@ +package com.provectus.kafka.ui.service.graphs; + +import java.time.Duration; +import java.util.Set; +import javax.annotation.Nullable; +import lombok.Builder; + +@Builder +public record GraphDescription(String id, + @Nullable Duration defaultInterval, //null for instant queries, set for range + String prometheusQuery, + Set params) { + + public static GraphDescriptionBuilder instant() { + return builder(); + } + + public static GraphDescriptionBuilder range(Duration defaultInterval) { + return builder().defaultInterval(defaultInterval); + } + + public boolean isRange() { + return defaultInterval != null; + } +} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/graphs/GraphDescriptions.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/graphs/GraphDescriptions.java new file mode 100644 index 00000000000..a6045e35859 --- /dev/null +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/graphs/GraphDescriptions.java @@ -0,0 +1,74 @@ +package com.provectus.kafka.ui.service.graphs; + +import static java.util.stream.Collectors.toMap; + +import com.provectus.kafka.ui.exception.ValidationException; +import java.time.Duration; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Stream; +import org.springframework.stereotype.Component; + +@Component +class GraphDescriptions { + + private static final Duration DEFAULT_RANGE_DURATION = Duration.ofDays(7); + + private final Map graphsById; + + GraphDescriptions() { + validate(); + this.graphsById = PREDEFINED_GRAPHS.stream().collect(toMap(GraphDescription::id, d -> d)); + } + + Optional getById(String id) { + return Optional.ofNullable(graphsById.get(id)); + } + + Stream all() { + return graphsById.values().stream(); + } + + private void validate() { + Map errors = new HashMap<>(); + for (GraphDescription description : PREDEFINED_GRAPHS) { + new PromQueryTemplate(description) + .validateSyntax() + .ifPresent(err -> errors.put(description.id(), err)); + } + if (!errors.isEmpty()) { + throw new ValidationException("Error validating queries for following graphs: " + errors); + } + } + + private static final List PREDEFINED_GRAPHS = List.of( + + GraphDescription.range(DEFAULT_RANGE_DURATION) + .id("broker_bytes_disk_ts") + .prometheusQuery("broker_bytes_disk{cluster=\"${cluster}\"}") + .params(Set.of()) + .build(), + + GraphDescription.instant() + .id("broker_bytes_disk") + .prometheusQuery("broker_bytes_disk{cluster=\"${cluster}\"}") + .params(Set.of()) + .build(), + + GraphDescription.instant() + .id("kafka_topic_partition_current_offset") + .prometheusQuery("kafka_topic_partition_current_offset{cluster=\"${cluster}\"}") + .params(Set.of()) + .build(), + + GraphDescription.range(DEFAULT_RANGE_DURATION) + .id("kafka_topic_partition_current_offset_per_topic_ts") + .prometheusQuery("kafka_topic_partition_current_offset{cluster=\"${cluster}\",topic = \"${topic}\"}") + .params(Set.of("topic")) + .build() + ); + +} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/graphs/GraphsService.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/graphs/GraphsService.java new file mode 100644 index 00000000000..5ac3965007c --- /dev/null +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/graphs/GraphsService.java @@ -0,0 +1,95 @@ +package com.provectus.kafka.ui.service.graphs; + +import com.google.common.base.Preconditions; +import com.provectus.kafka.ui.exception.NotFoundException; +import com.provectus.kafka.ui.exception.ValidationException; +import com.provectus.kafka.ui.model.KafkaCluster; +import java.time.Duration; +import java.time.Instant; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Stream; +import javax.annotation.Nullable; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; +import prometheus.query.api.PrometheusClientApi; +import prometheus.query.model.QueryResponse; +import reactor.core.publisher.Mono; + +@Component +@RequiredArgsConstructor +public class GraphsService { + + private static final int TARGET_MATRIX_DATA_POINTS = 200; + + private final GraphDescriptions graphDescriptions; + + public Mono getGraphData(KafkaCluster cluster, + String id, + @Nullable Instant from, + @Nullable Instant to, + @Nullable Map params) { + + var graph = graphDescriptions.getById(id) + .orElseThrow(() -> new NotFoundException("No graph found with id = " + id)); + + var promClient = cluster.getPrometheusStorageClient(); + if (promClient == null) { + throw new ValidationException("Prometheus not configured for cluster"); + } + String preparedQuery = prepareQuery(graph, cluster.getName(), params); + return cluster.getPrometheusStorageClient() + .mono(client -> { + if (graph.isRange()) { + return queryRange(client, preparedQuery, graph.defaultInterval(), from, to); + } + return queryInstant(client, preparedQuery); + }); + } + + private Mono queryRange(PrometheusClientApi c, + String preparedQuery, + Duration defaultPeriod, + @Nullable Instant from, + @Nullable Instant to) { + if (from == null) { + from = Instant.now().minus(defaultPeriod); + } + if (to == null) { + to = Instant.now(); + } + Preconditions.checkArgument(to.isAfter(from)); + return c.queryRange( + preparedQuery, + String.valueOf(from.getEpochSecond()), + String.valueOf(to.getEpochSecond()), + calculateStepSize(from, to), + null + ); + } + + private String calculateStepSize(Instant from, Instant to) { + long intervalInSecs = to.getEpochSecond() - from.getEpochSecond(); + if (intervalInSecs <= TARGET_MATRIX_DATA_POINTS) { + return intervalInSecs + "s"; + } + int step = ((int) (((double) intervalInSecs) / TARGET_MATRIX_DATA_POINTS)); + return step + "s"; + } + + private Mono queryInstant(PrometheusClientApi c, String preparedQuery) { + return c.query(preparedQuery, null, null); + } + + private String prepareQuery(GraphDescription d, String clusterName, @Nullable Map params) { + return new PromQueryTemplate(d).getQuery(clusterName, Optional.ofNullable(params).orElse(Map.of())); + } + + public Stream getGraphs(KafkaCluster cluster) { + if (cluster.getPrometheusStorageClient() == null) { + return Stream.empty(); + } + return graphDescriptions.all(); + } + +} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/graphs/PromQueryLangGrammar.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/graphs/PromQueryLangGrammar.java new file mode 100644 index 00000000000..072a3b9d7b9 --- /dev/null +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/graphs/PromQueryLangGrammar.java @@ -0,0 +1,35 @@ +package com.provectus.kafka.ui.service.graphs; + +import java.util.Optional; +import org.antlr.v4.runtime.BailErrorStrategy; +import org.antlr.v4.runtime.CharStreams; +import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.misc.ParseCancellationException; +import promql.PromQLLexer; +import promql.PromQLParser; + +class PromQueryLangGrammar { + + // returns error msg, or empty if query is valid + static Optional validateExpression(String query) { + try { + parseExpression(query); + return Optional.empty(); + } catch (ParseCancellationException e) { + //TODO: add more descriptive msg + return Optional.of("Syntax error"); + } + } + + static PromQLParser.ExpressionContext parseExpression(String query) { + return createParser(query).expression(); + } + + private static PromQLParser createParser(String str) { + var parser = new PromQLParser(new CommonTokenStream(new PromQLLexer(CharStreams.fromString(str)))); + parser.removeErrorListeners(); + parser.setErrorHandler(new BailErrorStrategy()); + return parser; + } + +} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/graphs/PromQueryTemplate.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/graphs/PromQueryTemplate.java new file mode 100644 index 00000000000..3f5b8ed058c --- /dev/null +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/graphs/PromQueryTemplate.java @@ -0,0 +1,51 @@ +package com.provectus.kafka.ui.service.graphs; + +import com.google.common.collect.Sets; +import com.provectus.kafka.ui.exception.ValidationException; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import org.apache.commons.lang3.text.StrSubstitutor; + +class PromQueryTemplate { + + private static final String CLUSTER_LABEL_NAME = "cluster"; + + private final String queryTemplate; + private final Set paramsNames; + + PromQueryTemplate(GraphDescription d) { + this(d.prometheusQuery(), d.params()); + } + + PromQueryTemplate(String templateQueryString, Set paramsNames) { + this.queryTemplate = templateQueryString; + this.paramsNames = paramsNames; + } + + String getQuery(String clusterName, Map paramValues) { + var missingParams = Sets.difference(paramsNames, paramValues.keySet()); + if (!missingParams.isEmpty()) { + throw new ValidationException("Not all params set for query, missing: " + missingParams); + } + Map replacements = new HashMap<>(paramValues); + replacements.put(CLUSTER_LABEL_NAME, clusterName); + return replaceParams(replacements); + } + + // returns error msg or empty if no errors found + Optional validateSyntax() { + Map fakeReplacements = new HashMap<>(); + fakeReplacements.put(CLUSTER_LABEL_NAME, "1"); + paramsNames.forEach(paramName -> fakeReplacements.put(paramName, "1")); + + String prepared = replaceParams(fakeReplacements); + return PromQueryLangGrammar.validateExpression(prepared); + } + + private String replaceParams(Map replacements) { + return new StrSubstitutor(replacements).replace(queryTemplate); + } + +} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/integration/odd/TopicsExporter.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/integration/odd/TopicsExporter.java index 8f4ef2781be..29947c6acc7 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/integration/odd/TopicsExporter.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/integration/odd/TopicsExporter.java @@ -1,8 +1,9 @@ package com.provectus.kafka.ui.service.integration.odd; +import static com.provectus.kafka.ui.service.metrics.scrape.ScrapedClusterState.TopicState; + import com.google.common.collect.ImmutableMap; import com.provectus.kafka.ui.model.KafkaCluster; -import com.provectus.kafka.ui.model.Statistics; import com.provectus.kafka.ui.service.StatisticsCache; import com.provectus.kafka.ui.service.integration.odd.schema.DataSetFieldsExtractors; import com.provectus.kafka.ui.sr.model.SchemaSubject; @@ -37,10 +38,10 @@ class TopicsExporter { Flux export(KafkaCluster cluster) { String clusterOddrn = Oddrn.clusterOddrn(cluster); - Statistics stats = statisticsCache.get(cluster); - return Flux.fromIterable(stats.getTopicDescriptions().keySet()) + var clusterState = statisticsCache.get(cluster).getClusterState(); + return Flux.fromIterable(clusterState.getTopicStates().keySet()) .filter(topicFilter) - .flatMap(topic -> createTopicDataEntity(cluster, topic, stats)) + .flatMap(topic -> createTopicDataEntity(cluster, topic, clusterState.getTopicStates().get(topic))) .onErrorContinue( (th, topic) -> log.warn("Error exporting data for topic {}, cluster {}", topic, cluster.getName(), th)) .buffer(100) @@ -50,7 +51,7 @@ Flux export(KafkaCluster cluster) { .items(topicsEntities)); } - private Mono createTopicDataEntity(KafkaCluster cluster, String topic, Statistics stats) { + private Mono createTopicDataEntity(KafkaCluster cluster, String topic, TopicState topicState) { KafkaPath topicOddrnPath = Oddrn.topicOddrnPath(cluster, topic); return Mono.zip( @@ -70,13 +71,13 @@ private Mono createTopicDataEntity(KafkaCluster cluster, String topi .addMetadataItem( new MetadataExtension() .schemaUrl(URI.create("wontbeused.oops")) - .metadata(getTopicMetadata(topic, stats))); + .metadata(getTopicMetadata(topicState))); } ); } - private Map getNonDefaultConfigs(String topic, Statistics stats) { - List config = stats.getTopicConfigs().get(topic); + private Map getNonDefaultConfigs(TopicState topicState) { + List config = topicState.configs(); if (config == null) { return Map.of(); } @@ -85,12 +86,12 @@ private Map getNonDefaultConfigs(String topic, Statistics stats) .collect(Collectors.toMap(ConfigEntry::name, ConfigEntry::value)); } - private Map getTopicMetadata(String topic, Statistics stats) { - TopicDescription topicDescription = stats.getTopicDescriptions().get(topic); + private Map getTopicMetadata(TopicState topicState) { + TopicDescription topicDescription = topicState.description(); return ImmutableMap.builder() .put("partitions", topicDescription.partitions().size()) .put("replication_factor", topicDescription.partitions().get(0).replicas().size()) - .putAll(getNonDefaultConfigs(topic, stats)) + .putAll(getNonDefaultConfigs(topicState)) .build(); } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/MetricsCollector.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/MetricsCollector.java deleted file mode 100644 index fca7ab1fea0..00000000000 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/MetricsCollector.java +++ /dev/null @@ -1,69 +0,0 @@ -package com.provectus.kafka.ui.service.metrics; - -import com.provectus.kafka.ui.model.KafkaCluster; -import com.provectus.kafka.ui.model.Metrics; -import com.provectus.kafka.ui.model.MetricsConfig; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.common.Node; -import org.springframework.stereotype.Component; -import reactor.core.publisher.Flux; -import reactor.core.publisher.Mono; -import reactor.util.function.Tuple2; -import reactor.util.function.Tuples; - -@Component -@Slf4j -@RequiredArgsConstructor -public class MetricsCollector { - - private final JmxMetricsRetriever jmxMetricsRetriever; - private final PrometheusMetricsRetriever prometheusMetricsRetriever; - - public Mono getBrokerMetrics(KafkaCluster cluster, Collection nodes) { - return Flux.fromIterable(nodes) - .flatMap(n -> getMetrics(cluster, n).map(lst -> Tuples.of(n, lst))) - .collectMap(Tuple2::getT1, Tuple2::getT2) - .map(nodeMetrics -> collectMetrics(cluster, nodeMetrics)) - .defaultIfEmpty(Metrics.empty()); - } - - private Mono> getMetrics(KafkaCluster kafkaCluster, Node node) { - Flux metricFlux = Flux.empty(); - if (kafkaCluster.getMetricsConfig() != null) { - String type = kafkaCluster.getMetricsConfig().getType(); - if (type == null || type.equalsIgnoreCase(MetricsConfig.JMX_METRICS_TYPE)) { - metricFlux = jmxMetricsRetriever.retrieve(kafkaCluster, node); - } else if (type.equalsIgnoreCase(MetricsConfig.PROMETHEUS_METRICS_TYPE)) { - metricFlux = prometheusMetricsRetriever.retrieve(kafkaCluster, node); - } - } - return metricFlux.collectList(); - } - - public Metrics collectMetrics(KafkaCluster cluster, Map> perBrokerMetrics) { - Metrics.MetricsBuilder builder = Metrics.builder() - .perBrokerMetrics( - perBrokerMetrics.entrySet() - .stream() - .collect(Collectors.toMap(e -> e.getKey().id(), Map.Entry::getValue))); - - populateWellknowMetrics(cluster, perBrokerMetrics) - .apply(builder); - - return builder.build(); - } - - private WellKnownMetrics populateWellknowMetrics(KafkaCluster cluster, Map> perBrokerMetrics) { - WellKnownMetrics wellKnownMetrics = new WellKnownMetrics(); - perBrokerMetrics.forEach((node, metrics) -> - metrics.forEach(metric -> - wellKnownMetrics.populate(node, metric))); - return wellKnownMetrics; - } - -} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/MetricsRetriever.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/MetricsRetriever.java deleted file mode 100644 index 7e1e126fa0d..00000000000 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/MetricsRetriever.java +++ /dev/null @@ -1,9 +0,0 @@ -package com.provectus.kafka.ui.service.metrics; - -import com.provectus.kafka.ui.model.KafkaCluster; -import org.apache.kafka.common.Node; -import reactor.core.publisher.Flux; - -interface MetricsRetriever { - Flux retrieve(KafkaCluster c, Node node); -} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/PrometheusEndpointMetricsParser.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/PrometheusEndpointMetricsParser.java deleted file mode 100644 index 1a51ca0afae..00000000000 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/PrometheusEndpointMetricsParser.java +++ /dev/null @@ -1,46 +0,0 @@ -package com.provectus.kafka.ui.service.metrics; - -import java.math.BigDecimal; -import java.util.Arrays; -import java.util.Optional; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.math.NumberUtils; - -@Slf4j -class PrometheusEndpointMetricsParser { - - /** - * Matches openmetrics format. For example, string: - * kafka_server_BrokerTopicMetrics_FiveMinuteRate{name="BytesInPerSec",topic="__consumer_offsets",} 16.94886650744339 - * will produce: - * name=kafka_server_BrokerTopicMetrics_FiveMinuteRate - * value=16.94886650744339 - * labels={name="BytesInPerSec", topic="__consumer_offsets"}", - */ - private static final Pattern PATTERN = Pattern.compile( - "(?^\\w+)([ \t]*\\{*(?.*)}*)[ \\t]+(?[\\d]+\\.?[\\d]+)?"); - - static Optional parse(String s) { - Matcher matcher = PATTERN.matcher(s); - if (matcher.matches()) { - String value = matcher.group("value"); - String metricName = matcher.group("metricName"); - if (metricName == null || !NumberUtils.isCreatable(value)) { - return Optional.empty(); - } - var labels = Arrays.stream(matcher.group("properties").split(",")) - .filter(str -> !"".equals(str)) - .map(str -> str.split("=")) - .filter(spit -> spit.length == 2) - .collect(Collectors.toUnmodifiableMap( - str -> str[0].trim(), - str -> str[1].trim().replace("\"", ""))); - - return Optional.of(RawMetric.create(metricName, labels, new BigDecimal(value))); - } - return Optional.empty(); - } -} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/PrometheusMetricsRetriever.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/PrometheusMetricsRetriever.java deleted file mode 100644 index 33ef1b80723..00000000000 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/PrometheusMetricsRetriever.java +++ /dev/null @@ -1,70 +0,0 @@ -package com.provectus.kafka.ui.service.metrics; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Strings; -import com.provectus.kafka.ui.config.ClustersProperties; -import com.provectus.kafka.ui.model.KafkaCluster; -import com.provectus.kafka.ui.model.MetricsConfig; -import com.provectus.kafka.ui.util.WebClientConfigurator; -import java.util.Arrays; -import java.util.Optional; -import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.common.Node; -import org.springframework.stereotype.Service; -import org.springframework.util.unit.DataSize; -import org.springframework.web.reactive.function.client.WebClient; -import org.springframework.web.util.UriComponentsBuilder; -import reactor.core.publisher.Flux; -import reactor.core.publisher.Mono; - -@Service -@Slf4j -class PrometheusMetricsRetriever implements MetricsRetriever { - - private static final String METRICS_ENDPOINT_PATH = "/metrics"; - private static final int DEFAULT_EXPORTER_PORT = 11001; - - @Override - public Flux retrieve(KafkaCluster c, Node node) { - log.debug("Retrieving metrics from prometheus exporter: {}:{}", node.host(), c.getMetricsConfig().getPort()); - - MetricsConfig metricsConfig = c.getMetricsConfig(); - var webClient = new WebClientConfigurator() - .configureBufferSize(DataSize.ofMegabytes(20)) - .configureBasicAuth(metricsConfig.getUsername(), metricsConfig.getPassword()) - .configureSsl( - c.getOriginalProperties().getSsl(), - new ClustersProperties.KeystoreConfig( - metricsConfig.getKeystoreLocation(), - metricsConfig.getKeystorePassword())) - .build(); - - return retrieve(webClient, node.host(), c.getMetricsConfig()); - } - - @VisibleForTesting - Flux retrieve(WebClient webClient, String host, MetricsConfig metricsConfig) { - int port = Optional.ofNullable(metricsConfig.getPort()).orElse(DEFAULT_EXPORTER_PORT); - boolean sslEnabled = metricsConfig.isSsl() || metricsConfig.getKeystoreLocation() != null; - var request = webClient.get() - .uri(UriComponentsBuilder.newInstance() - .scheme(sslEnabled ? "https" : "http") - .host(host) - .port(port) - .path(METRICS_ENDPOINT_PATH).build().toUri()); - - WebClient.ResponseSpec responseSpec = request.retrieve(); - return responseSpec.bodyToMono(String.class) - .doOnError(e -> log.error("Error while getting metrics from {}", host, e)) - .onErrorResume(th -> Mono.empty()) - .flatMapMany(body -> - Flux.fromStream( - Arrays.stream(body.split("\\n")) - .filter(str -> !Strings.isNullOrEmpty(str) && !str.startsWith("#")) // skipping comments strings - .map(PrometheusEndpointMetricsParser::parse) - .filter(Optional::isPresent) - .map(Optional::get) - ) - ); - } -} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/RawMetric.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/RawMetric.java index 659212f23ff..d26e5785841 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/RawMetric.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/RawMetric.java @@ -1,10 +1,15 @@ package com.provectus.kafka.ui.service.metrics; +import static io.prometheus.client.Collector.MetricFamilySamples; +import static io.prometheus.client.Collector.Type; + import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; -import lombok.AllArgsConstructor; -import lombok.EqualsAndHashCode; -import lombok.ToString; +import java.util.stream.Stream; public interface RawMetric { @@ -14,47 +19,27 @@ public interface RawMetric { BigDecimal value(); - // Key, that can be used for metrics reductions - default Object identityKey() { - return name() + "_" + labels(); - } - - RawMetric copyWithValue(BigDecimal newValue); - //-------------------------------------------------- static RawMetric create(String name, Map labels, BigDecimal value) { return new SimpleMetric(name, labels, value); } - @AllArgsConstructor - @EqualsAndHashCode - @ToString - class SimpleMetric implements RawMetric { - - private final String name; - private final Map labels; - private final BigDecimal value; - - @Override - public String name() { - return name; - } - - @Override - public Map labels() { - return labels; - } - - @Override - public BigDecimal value() { - return value; - } - - @Override - public RawMetric copyWithValue(BigDecimal newValue) { - return new SimpleMetric(name, labels, newValue); + static Stream groupIntoMfs(Collection rawMetrics) { + Map map = new LinkedHashMap<>(); + for (RawMetric m : rawMetrics) { + var mfs = map.get(m.name()); + if (mfs == null) { + mfs = new MetricFamilySamples(m.name(), Type.GAUGE, m.name(), new ArrayList<>()); + map.put(m.name(), mfs); + } + List lbls = m.labels().keySet().stream().toList(); + List lblVals = lbls.stream().map(l -> m.labels().get(l)).toList(); + mfs.samples.add(new MetricFamilySamples.Sample(m.name(), lbls, lblVals, m.value().doubleValue())); } + return map.values().stream(); } + record SimpleMetric(String name, Map labels, BigDecimal value) implements RawMetric { } + } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/SummarizedMetrics.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/SummarizedMetrics.java new file mode 100644 index 00000000000..47bc65beeb0 --- /dev/null +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/SummarizedMetrics.java @@ -0,0 +1,73 @@ +package com.provectus.kafka.ui.service.metrics; + +import static java.util.stream.Collectors.toMap; + +import com.google.common.collect.Streams; +import com.provectus.kafka.ui.model.Metrics; +import groovy.lang.Tuple; +import io.prometheus.client.Collector.MetricFamilySamples; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.Optional; +import java.util.stream.Stream; +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor +public class SummarizedMetrics { + + private final Metrics metrics; + + public Stream asStream() { + return Streams.concat( + metrics.getInferredMetrics().asStream(), + metrics.getPerBrokerScrapedMetrics() + .values() + .stream() + .flatMap(Collection::stream) + .collect(toMap(mfs -> mfs.name, Optional::of, SummarizedMetrics::summarizeMfs, LinkedHashMap::new)) + .values() + .stream() + .filter(Optional::isPresent) + .map(Optional::get) + ); + } + + //returns Optional.empty if merging not supported for metric type + private static Optional summarizeMfs(Optional mfs1opt, + Optional mfs2opt) { + if ((mfs1opt.isEmpty() || mfs2opt.isEmpty()) || (mfs1opt.get().type != mfs2opt.get().type)) { + return Optional.empty(); + } + var mfs1 = mfs1opt.get(); + return switch (mfs1.type) { + case GAUGE, COUNTER -> Optional.of( + new MetricFamilySamples( + mfs1.name, + mfs1.type, + mfs1.help, + Stream.concat(mfs1.samples.stream(), mfs2opt.get().samples.stream()) + .collect( + toMap( + // merging samples with same labels + s -> Tuple.tuple(s.name, s.labelNames, s.labelValues), + s -> s, + (s1, s2) -> new MetricFamilySamples.Sample( + s1.name, + s1.labelNames, + s1.labelValues, + s1.value + s2.value + ), + LinkedHashMap::new + ) + ) + .values() + .stream() + .toList() + ) + ); + default -> Optional.empty(); + }; + } + + +} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/WellKnownMetrics.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/WellKnownMetrics.java deleted file mode 100644 index 10ad128c657..00000000000 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/WellKnownMetrics.java +++ /dev/null @@ -1,67 +0,0 @@ -package com.provectus.kafka.ui.service.metrics; - -import static org.apache.commons.lang3.StringUtils.containsIgnoreCase; -import static org.apache.commons.lang3.StringUtils.endsWithIgnoreCase; - -import com.provectus.kafka.ui.model.Metrics; -import java.math.BigDecimal; -import java.util.HashMap; -import java.util.Map; -import org.apache.kafka.common.Node; - -class WellKnownMetrics { - - // per broker - final Map brokerBytesInFifteenMinuteRate = new HashMap<>(); - final Map brokerBytesOutFifteenMinuteRate = new HashMap<>(); - - // per topic - final Map bytesInFifteenMinuteRate = new HashMap<>(); - final Map bytesOutFifteenMinuteRate = new HashMap<>(); - - void populate(Node node, RawMetric rawMetric) { - updateBrokerIOrates(node, rawMetric); - updateTopicsIOrates(rawMetric); - } - - void apply(Metrics.MetricsBuilder metricsBuilder) { - metricsBuilder.topicBytesInPerSec(bytesInFifteenMinuteRate); - metricsBuilder.topicBytesOutPerSec(bytesOutFifteenMinuteRate); - metricsBuilder.brokerBytesInPerSec(brokerBytesInFifteenMinuteRate); - metricsBuilder.brokerBytesOutPerSec(brokerBytesOutFifteenMinuteRate); - } - - private void updateBrokerIOrates(Node node, RawMetric rawMetric) { - String name = rawMetric.name(); - if (!brokerBytesInFifteenMinuteRate.containsKey(node.id()) - && rawMetric.labels().size() == 1 - && "BytesInPerSec".equalsIgnoreCase(rawMetric.labels().get("name")) - && containsIgnoreCase(name, "BrokerTopicMetrics") - && endsWithIgnoreCase(name, "FifteenMinuteRate")) { - brokerBytesInFifteenMinuteRate.put(node.id(), rawMetric.value()); - } - if (!brokerBytesOutFifteenMinuteRate.containsKey(node.id()) - && rawMetric.labels().size() == 1 - && "BytesOutPerSec".equalsIgnoreCase(rawMetric.labels().get("name")) - && containsIgnoreCase(name, "BrokerTopicMetrics") - && endsWithIgnoreCase(name, "FifteenMinuteRate")) { - brokerBytesOutFifteenMinuteRate.put(node.id(), rawMetric.value()); - } - } - - private void updateTopicsIOrates(RawMetric rawMetric) { - String name = rawMetric.name(); - String topic = rawMetric.labels().get("topic"); - if (topic != null - && containsIgnoreCase(name, "BrokerTopicMetrics") - && endsWithIgnoreCase(name, "FifteenMinuteRate")) { - String nameProperty = rawMetric.labels().get("name"); - if ("BytesInPerSec".equalsIgnoreCase(nameProperty)) { - bytesInFifteenMinuteRate.compute(topic, (k, v) -> v == null ? rawMetric.value() : v.add(rawMetric.value())); - } else if ("BytesOutPerSec".equalsIgnoreCase(nameProperty)) { - bytesOutFifteenMinuteRate.compute(topic, (k, v) -> v == null ? rawMetric.value() : v.add(rawMetric.value())); - } - } - } - -} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/prometheus/PrometheusExpose.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/prometheus/PrometheusExpose.java new file mode 100644 index 00000000000..2a1464a1bfc --- /dev/null +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/prometheus/PrometheusExpose.java @@ -0,0 +1,123 @@ +package com.provectus.kafka.ui.service.metrics.prometheus; + +import static io.prometheus.client.Collector.MetricFamilySamples; +import static io.prometheus.client.exporter.common.TextFormat.CONTENT_TYPE_OPENMETRICS_100; +import static org.springframework.http.HttpHeaders.CONTENT_TYPE; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Iterators; +import com.provectus.kafka.ui.model.Metrics; +import io.prometheus.client.exporter.common.TextFormat; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import lombok.SneakyThrows; +import org.springframework.http.HttpHeaders; +import org.springframework.http.ResponseEntity; + +public final class PrometheusExpose { + + private static final String CLUSTER_EXPOSE_LBL_NAME = "cluster"; + private static final String BROKER_EXPOSE_LBL_NAME = "broker_id"; + + private static final HttpHeaders PROMETHEUS_EXPOSE_ENDPOINT_HEADERS; + + static { + PROMETHEUS_EXPOSE_ENDPOINT_HEADERS = new HttpHeaders(); + PROMETHEUS_EXPOSE_ENDPOINT_HEADERS.set(CONTENT_TYPE, CONTENT_TYPE_OPENMETRICS_100); + } + + private PrometheusExpose() { + } + + public static ResponseEntity exposeAllMetrics(Map clustersMetrics) { + return constructHttpsResponse(getMetricsForGlobalExpose(clustersMetrics)); + } + + private static Stream getMetricsForGlobalExpose(Map clustersMetrics) { + return clustersMetrics.entrySet() + .stream() + .flatMap(e -> prepareMetricsForGlobalExpose(e.getKey(), e.getValue())) + // merging MFS with same name with LinkedHashMap(for order keeping) + .collect(Collectors.toMap(mfs -> mfs.name, mfs -> mfs, + PrometheusExpose::concatSamples, LinkedHashMap::new)) + .values() + .stream(); + } + + public static Stream prepareMetricsForGlobalExpose(String clusterName, Metrics metrics) { + return Stream.concat( + metrics.getInferredMetrics().asStream(), + extractBrokerMetricsWithLabel(metrics) + ) + .map(mfs -> appendLabel(mfs, CLUSTER_EXPOSE_LBL_NAME, clusterName)); + } + + private static Stream extractBrokerMetricsWithLabel(Metrics metrics) { + return metrics.getPerBrokerScrapedMetrics().entrySet().stream() + .flatMap(e -> { + String brokerId = String.valueOf(e.getKey()); + return e.getValue().stream().map(mfs -> appendLabel(mfs, BROKER_EXPOSE_LBL_NAME, brokerId)); + }); + } + + private static MetricFamilySamples concatSamples(MetricFamilySamples mfs1, + MetricFamilySamples mfs2) { + return new MetricFamilySamples( + mfs1.name, mfs1.unit, mfs1.type, mfs1.help, + Stream.concat(mfs1.samples.stream(), mfs2.samples.stream()).toList() + ); + } + + private static MetricFamilySamples appendLabel(MetricFamilySamples mfs, String lblName, String lblVal) { + return new MetricFamilySamples( + mfs.name, mfs.unit, mfs.type, mfs.help, + mfs.samples.stream() + .map(sample -> + new MetricFamilySamples.Sample( + sample.name, + prependToList(sample.labelNames, lblName), + prependToList(sample.labelValues, lblVal), + sample.value + )).toList() + ); + } + + private static List prependToList(List lst, T toPrepend) { + var result = new ArrayList(lst.size() + 1); + result.add(toPrepend); + result.addAll(lst); + return result; + } + + @VisibleForTesting + @SneakyThrows + public static ResponseEntity constructHttpsResponse(Stream metrics) { + StringWriter writer = new StringWriter(); + TextFormat.writeOpenMetrics100(writer, Iterators.asEnumeration(metrics.iterator())); + return ResponseEntity + .ok() + .headers(PROMETHEUS_EXPOSE_ENDPOINT_HEADERS) + .body(writer.toString()); + } + + // copied from io.prometheus.client.exporter.common.TextFormat:writeEscapedLabelValue + public static String escapedLabelValue(String s) { + StringBuilder sb = new StringBuilder(s.length()); + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + switch (c) { + case '\\' -> sb.append("\\\\"); + case '\"' -> sb.append("\\\""); + case '\n' -> sb.append("\\n"); + default -> sb.append(c); + } + } + return sb.toString(); + } + +} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/IoRatesMetricsScanner.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/IoRatesMetricsScanner.java new file mode 100644 index 00000000000..b0935d8a214 --- /dev/null +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/IoRatesMetricsScanner.java @@ -0,0 +1,83 @@ +package com.provectus.kafka.ui.service.metrics.scrape; + +import static io.prometheus.client.Collector.MetricFamilySamples; +import static org.apache.commons.lang3.StringUtils.containsIgnoreCase; +import static org.apache.commons.lang3.StringUtils.endsWithIgnoreCase; + +import com.provectus.kafka.ui.model.Metrics; +import java.math.BigDecimal; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +// Scans external jmx/prometheus metric and tries to infer io rates +class IoRatesMetricsScanner { + + // per broker + final Map brokerBytesInFifteenMinuteRate = new HashMap<>(); + final Map brokerBytesOutFifteenMinuteRate = new HashMap<>(); + + // per topic + final Map bytesInFifteenMinuteRate = new HashMap<>(); + final Map bytesOutFifteenMinuteRate = new HashMap<>(); + + IoRatesMetricsScanner(Map> perBrokerMetrics) { + perBrokerMetrics.forEach((nodeId, metrics) -> { + metrics.forEach(m -> { + m.samples.forEach(metricSample -> { + updateBrokerIOrates(nodeId, metricSample); + updateTopicsIOrates(metricSample); + }); + }); + }); + } + + Metrics.IoRates get() { + return Metrics.IoRates.builder() + .topicBytesInPerSec(bytesInFifteenMinuteRate) + .topicBytesOutPerSec(bytesOutFifteenMinuteRate) + .brokerBytesInPerSec(brokerBytesInFifteenMinuteRate) + .brokerBytesOutPerSec(brokerBytesOutFifteenMinuteRate) + .build(); + } + + private void updateBrokerIOrates(int nodeId, MetricFamilySamples.Sample metric) { + String name = metric.name; + if (!brokerBytesInFifteenMinuteRate.containsKey(nodeId) + && metric.labelValues.size() == 1 + && "BytesInPerSec".equalsIgnoreCase(metric.labelValues.get(0)) + && containsIgnoreCase(name, "BrokerTopicMetrics") + && endsWithIgnoreCase(name, "FifteenMinuteRate")) { + brokerBytesInFifteenMinuteRate.put(nodeId, BigDecimal.valueOf(metric.value)); + } + if (!brokerBytesOutFifteenMinuteRate.containsKey(nodeId) + && metric.labelValues.size() == 1 + && "BytesOutPerSec".equalsIgnoreCase(metric.labelValues.get(0)) + && containsIgnoreCase(name, "BrokerTopicMetrics") + && endsWithIgnoreCase(name, "FifteenMinuteRate")) { + brokerBytesOutFifteenMinuteRate.put(nodeId, BigDecimal.valueOf(metric.value)); + } + } + + private void updateTopicsIOrates(MetricFamilySamples.Sample metric) { + String name = metric.name; + int topicLblIdx = metric.labelNames.indexOf("topic"); + if (topicLblIdx >= 0 + && containsIgnoreCase(name, "BrokerTopicMetrics") + && endsWithIgnoreCase(name, "FifteenMinuteRate")) { + String topic = metric.labelValues.get(topicLblIdx); + int nameLblIdx = metric.labelNames.indexOf("name"); + if (nameLblIdx >= 0) { + var nameLblVal = metric.labelValues.get(nameLblIdx); + if ("BytesInPerSec".equalsIgnoreCase(nameLblVal)) { + BigDecimal val = BigDecimal.valueOf(metric.value); + bytesInFifteenMinuteRate.merge(topic, val, BigDecimal::add); + } else if ("BytesOutPerSec".equalsIgnoreCase(nameLblVal)) { + BigDecimal val = BigDecimal.valueOf(metric.value); + bytesOutFifteenMinuteRate.merge(topic, val, BigDecimal::add); + } + } + } + } + +} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/MetricsScrapping.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/MetricsScrapping.java new file mode 100644 index 00000000000..42d7dcd9948 --- /dev/null +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/MetricsScrapping.java @@ -0,0 +1,94 @@ +package com.provectus.kafka.ui.service.metrics.scrape; + +import static com.provectus.kafka.ui.config.ClustersProperties.Cluster; +import static com.provectus.kafka.ui.config.ClustersProperties.KeystoreConfig; +import static com.provectus.kafka.ui.model.MetricsScrapeProperties.JMX_METRICS_TYPE; +import static com.provectus.kafka.ui.model.MetricsScrapeProperties.PROMETHEUS_METRICS_TYPE; +import static io.prometheus.client.Collector.MetricFamilySamples; + +import com.provectus.kafka.ui.model.Metrics; +import com.provectus.kafka.ui.model.MetricsScrapeProperties; +import com.provectus.kafka.ui.service.metrics.prometheus.PrometheusExpose; +import com.provectus.kafka.ui.service.metrics.scrape.inferred.InferredMetrics; +import com.provectus.kafka.ui.service.metrics.scrape.inferred.InferredMetricsScraper; +import com.provectus.kafka.ui.service.metrics.scrape.jmx.JmxMetricsRetriever; +import com.provectus.kafka.ui.service.metrics.scrape.jmx.JmxMetricsScraper; +import com.provectus.kafka.ui.service.metrics.scrape.prometheus.PrometheusScraper; +import com.provectus.kafka.ui.service.metrics.sink.MetricsSink; +import jakarta.annotation.Nullable; +import java.util.Collection; +import java.util.Optional; +import java.util.stream.Stream; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.common.Node; +import reactor.core.publisher.Mono; + +@Slf4j +@RequiredArgsConstructor +public class MetricsScrapping { + + private final String clusterName; + private final MetricsSink sink; + private final InferredMetricsScraper inferredMetricsScraper; + @Nullable + private final JmxMetricsScraper jmxMetricsScraper; + @Nullable + private final PrometheusScraper prometheusScraper; + + public static MetricsScrapping create(Cluster cluster, + JmxMetricsRetriever jmxMetricsRetriever) { + JmxMetricsScraper jmxMetricsScraper = null; + PrometheusScraper prometheusScraper = null; + var metrics = cluster.getMetrics(); + if (cluster.getMetrics() != null) { + var scrapeProperties = MetricsScrapeProperties.create(cluster); + if (metrics.getType().equalsIgnoreCase(JMX_METRICS_TYPE) && metrics.getPort() != null) { + jmxMetricsScraper = new JmxMetricsScraper(scrapeProperties, jmxMetricsRetriever); + } else if (metrics.getType().equalsIgnoreCase(PROMETHEUS_METRICS_TYPE)) { + prometheusScraper = new PrometheusScraper(scrapeProperties); + } + } + return new MetricsScrapping( + cluster.getName(), + MetricsSink.create(cluster), + new InferredMetricsScraper(), + jmxMetricsScraper, + prometheusScraper + ); + } + + public Mono scrape(ScrapedClusterState clusterState, Collection nodes) { + Mono inferred = inferredMetricsScraper.scrape(clusterState); + Mono external = scrapeExternal(nodes); + return inferred.zipWith( + external, + (inf, ext) -> Metrics.builder() + .inferredMetrics(inf) + .ioRates(ext.ioRates()) + .perBrokerScrapedMetrics(ext.perBrokerMetrics()) + .build() + ).doOnNext(this::sendMetricsToSink); + } + + private void sendMetricsToSink(Metrics metrics) { + sink.send(prepareMetricsForSending(metrics)) + .doOnError(th -> log.warn("Error sending metrics to metrics sink", th)) + .subscribe(); + } + + private Stream prepareMetricsForSending(Metrics metrics) { + return PrometheusExpose.prepareMetricsForGlobalExpose(clusterName, metrics); + } + + private Mono scrapeExternal(Collection nodes) { + if (jmxMetricsScraper != null) { + return jmxMetricsScraper.scrape(nodes); + } + if (prometheusScraper != null) { + return prometheusScraper.scrape(nodes); + } + return Mono.just(PerBrokerScrapedMetrics.empty()); + } + +} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/PerBrokerScrapedMetrics.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/PerBrokerScrapedMetrics.java new file mode 100644 index 00000000000..6cfd33241c1 --- /dev/null +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/PerBrokerScrapedMetrics.java @@ -0,0 +1,19 @@ +package com.provectus.kafka.ui.service.metrics.scrape; + +import static io.prometheus.client.Collector.MetricFamilySamples; + +import com.provectus.kafka.ui.model.Metrics; +import java.util.List; +import java.util.Map; + +public record PerBrokerScrapedMetrics(Map> perBrokerMetrics) { + + static PerBrokerScrapedMetrics empty() { + return new PerBrokerScrapedMetrics(Map.of()); + } + + Metrics.IoRates ioRates() { + return new IoRatesMetricsScanner(perBrokerMetrics).get(); + } + +} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/ScrapedClusterState.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/ScrapedClusterState.java new file mode 100644 index 00000000000..ac5587193dc --- /dev/null +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/ScrapedClusterState.java @@ -0,0 +1,198 @@ +package com.provectus.kafka.ui.service.metrics.scrape; + +import static com.provectus.kafka.ui.model.InternalLogDirStats.LogDirSpaceStats; +import static com.provectus.kafka.ui.model.InternalLogDirStats.SegmentStats; +import static com.provectus.kafka.ui.service.ReactiveAdminClient.ClusterDescription; + +import com.google.common.collect.Table; +import com.provectus.kafka.ui.model.InternalLogDirStats; +import com.provectus.kafka.ui.model.InternalPartitionsOffsets; +import com.provectus.kafka.ui.service.ReactiveAdminClient; +import jakarta.annotation.Nullable; +import java.time.Instant; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import lombok.Builder; +import lombok.RequiredArgsConstructor; +import lombok.Value; +import org.apache.kafka.clients.admin.ConfigEntry; +import org.apache.kafka.clients.admin.ConsumerGroupDescription; +import org.apache.kafka.clients.admin.ConsumerGroupListing; +import org.apache.kafka.clients.admin.OffsetSpec; +import org.apache.kafka.clients.admin.TopicDescription; +import org.apache.kafka.common.Node; +import org.apache.kafka.common.TopicPartition; +import reactor.core.publisher.Mono; + +@Builder(toBuilder = true) +@RequiredArgsConstructor +@Value +public class ScrapedClusterState { + + Instant scrapeFinishedAt; + Map nodesStates; + Map topicStates; + Map consumerGroupsStates; + + public record NodeState(int id, + Node node, + @Nullable SegmentStats segmentStats, + @Nullable LogDirSpaceStats logDirSpaceStats) { + } + + public record TopicState( + String name, + TopicDescription description, + List configs, + Map startOffsets, + Map endOffsets, + @Nullable SegmentStats segmentStats, + @Nullable Map partitionsSegmentStats) { + } + + public record ConsumerGroupState( + String group, + ConsumerGroupDescription description, + Map committedOffsets) { + } + + public static ScrapedClusterState empty() { + return ScrapedClusterState.builder() + .scrapeFinishedAt(Instant.now()) + .nodesStates(Map.of()) + .topicStates(Map.of()) + .consumerGroupsStates(Map.of()) + .build(); + } + + public ScrapedClusterState updateTopics(Map descriptions, + Map> configs, + InternalPartitionsOffsets partitionsOffsets) { + var updatedTopicStates = new HashMap<>(topicStates); + descriptions.forEach((topic, description) -> { + SegmentStats segmentStats = null; + Map partitionsSegmentStats = null; + if (topicStates.containsKey(topic)) { + segmentStats = topicStates.get(topic).segmentStats(); + partitionsSegmentStats = topicStates.get(topic).partitionsSegmentStats(); + } + updatedTopicStates.put( + topic, + new TopicState( + topic, + description, + configs.getOrDefault(topic, List.of()), + partitionsOffsets.topicOffsets(topic, true), + partitionsOffsets.topicOffsets(topic, false), + segmentStats, + partitionsSegmentStats + ) + ); + }); + return toBuilder() + .topicStates(updatedTopicStates) + .build(); + } + + public ScrapedClusterState topicDeleted(String topic) { + var newTopicStates = new HashMap<>(topicStates); + newTopicStates.remove(topic); + return toBuilder() + .topicStates(newTopicStates) + .build(); + } + + public static Mono scrape(ClusterDescription clusterDescription, + ReactiveAdminClient ac) { + return Mono.zip( + ac.describeLogDirs(clusterDescription.getNodes().stream().map(Node::id).toList()) + .map(InternalLogDirStats::new), + ac.listConsumerGroups().map(l -> l.stream().map(ConsumerGroupListing::groupId).toList()), + ac.describeTopics(), + ac.getTopicsConfig() + ).flatMap(phase1 -> + Mono.zip( + ac.listOffsets(phase1.getT3().values(), OffsetSpec.latest()), + ac.listOffsets(phase1.getT3().values(), OffsetSpec.earliest()), + ac.describeConsumerGroups(phase1.getT2()), + ac.listConsumerGroupOffsets(phase1.getT2(), null) + ).map(phase2 -> + create( + clusterDescription, + phase1.getT1(), + phase1.getT3(), + phase1.getT4(), + phase2.getT1(), + phase2.getT2(), + phase2.getT3(), + phase2.getT4() + ))); + } + + private static ScrapedClusterState create(ClusterDescription clusterDescription, + InternalLogDirStats segmentStats, + Map topicDescriptions, + Map> topicConfigs, + Map latestOffsets, + Map earliestOffsets, + Map consumerDescriptions, + Table consumerOffsets) { + + + Map topicStates = new HashMap<>(); + topicDescriptions.forEach((name, desc) -> + topicStates.put( + name, + new TopicState( + name, + desc, + topicConfigs.getOrDefault(name, List.of()), + filterTopic(name, earliestOffsets), + filterTopic(name, latestOffsets), + segmentStats.getTopicStats().get(name), + Optional.ofNullable(segmentStats.getPartitionsStats()) + .map(topicForFilter -> filterTopic(name, topicForFilter)) + .orElse(null) + ))); + + Map consumerGroupsStates = new HashMap<>(); + consumerDescriptions.forEach((name, desc) -> + consumerGroupsStates.put( + name, + new ConsumerGroupState( + name, + desc, + consumerOffsets.row(name) + ))); + + Map nodesStates = new HashMap<>(); + clusterDescription.getNodes().forEach(node -> + nodesStates.put( + node.id(), + new NodeState( + node.id(), + node, + segmentStats.getBrokerStats().get(node.id()), + segmentStats.getBrokerDirsStats().get(node.id()) + ))); + + return new ScrapedClusterState( + Instant.now(), + nodesStates, + topicStates, + consumerGroupsStates + ); + } + + private static Map filterTopic(String topicForFilter, Map tpMap) { + return tpMap.entrySet() + .stream() + .filter(tp -> tp.getKey().topic().equals(topicForFilter)) + .collect(Collectors.toMap(e -> e.getKey().partition(), Map.Entry::getValue)); + } + + +} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/inferred/InferredMetrics.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/inferred/InferredMetrics.java new file mode 100644 index 00000000000..8321bd149af --- /dev/null +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/inferred/InferredMetrics.java @@ -0,0 +1,24 @@ +package com.provectus.kafka.ui.service.metrics.scrape.inferred; + +import static io.prometheus.client.Collector.MetricFamilySamples; + +import java.util.List; +import java.util.stream.Stream; + +public class InferredMetrics { + + private final List metrics; + + public static InferredMetrics empty() { + return new InferredMetrics(List.of()); + } + + public InferredMetrics(List metrics) { + this.metrics = metrics; + } + + public Stream asStream() { + return metrics.stream(); + } + +} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/inferred/InferredMetricsScraper.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/inferred/InferredMetricsScraper.java new file mode 100644 index 00000000000..aaa38c290ba --- /dev/null +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/inferred/InferredMetricsScraper.java @@ -0,0 +1,226 @@ +package com.provectus.kafka.ui.service.metrics.scrape.inferred; + +import com.google.common.annotations.VisibleForTesting; +import com.provectus.kafka.ui.service.metrics.scrape.ScrapedClusterState; +import io.prometheus.client.Collector.MetricFamilySamples; +import io.prometheus.client.GaugeMetricFamily; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import javax.annotation.Nullable; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.admin.MemberDescription; +import org.apache.kafka.common.Node; +import reactor.core.publisher.Mono; + +@Slf4j +@RequiredArgsConstructor +public class InferredMetricsScraper { + + private ScrapedClusterState prevState = null; + + public synchronized Mono scrape(ScrapedClusterState newState) { + var inferred = infer(prevState, newState); + this.prevState = newState; + return Mono.just(inferred); + } + + @VisibleForTesting + static InferredMetrics infer(@Nullable ScrapedClusterState prevState, ScrapedClusterState newState) { + var registry = new MetricsRegistry(); + fillNodesMetrics(registry, newState); + fillTopicMetrics(registry, newState); + fillConsumerGroupsMetrics(registry, newState); + List metrics = registry.metrics.values().stream().toList(); + log.debug("{} metric families inferred from cluster state", metrics.size()); + return new InferredMetrics(metrics); + } + + private static class MetricsRegistry { + + final Map metrics = new LinkedHashMap<>(); + + void gauge(String name, + String help, + List lbls, + List lblVals, + Number value) { + GaugeMetricFamily gauge; + if ((gauge = (GaugeMetricFamily) metrics.get(name)) == null) { + gauge = new GaugeMetricFamily(name, help, lbls); + metrics.put(name, gauge); + } + gauge.addMetric(lblVals, value.doubleValue()); + } + } + + private static void fillNodesMetrics(MetricsRegistry registry, ScrapedClusterState newState) { + registry.gauge( + "broker_count", + "Number of brokers in the Kafka cluster", + List.of(), + List.of(), + newState.getNodesStates().size() + ); + + newState.getNodesStates().forEach((nodeId, state) -> { + if (state.segmentStats() != null) { + registry.gauge( + "broker_bytes_disk", + "Written disk size in bytes of a broker", + List.of("node_id"), + List.of(nodeId.toString()), + state.segmentStats().getSegmentSize() + ); + } + if (state.logDirSpaceStats() != null) { + if (state.logDirSpaceStats().usableBytes() != null) { + registry.gauge( + "broker_bytes_usable", + "Usable disk size in bytes of a broker", + List.of("node_id"), + List.of(nodeId.toString()), + state.logDirSpaceStats().usableBytes() + ); + } + if (state.logDirSpaceStats().totalBytes() != null) { + registry.gauge( + "broker_bytes_total", + "Total disk size in bytes of a broker", + List.of("node_id"), + List.of(nodeId.toString()), + state.logDirSpaceStats().totalBytes() + ); + } + } + }); + } + + private static void fillTopicMetrics(MetricsRegistry registry, ScrapedClusterState clusterState) { + registry.gauge( + "topic_count", + "Number of topics in the Kafka cluster", + List.of(), + List.of(), + clusterState.getTopicStates().size() + ); + + clusterState.getTopicStates().forEach((topicName, state) -> { + registry.gauge( + "kafka_topic_partitions", + "Number of partitions for this Topic", + List.of("topic"), + List.of(topicName), + state.description().partitions().size() + ); + state.endOffsets().forEach((partition, endOffset) -> { + registry.gauge( + "kafka_topic_partition_current_offset", + "Current Offset of a Broker at Topic/Partition", + List.of("topic", "partition"), + List.of(topicName, String.valueOf(partition)), + endOffset + ); + }); + state.startOffsets().forEach((partition, startOffset) -> { + registry.gauge( + "kafka_topic_partition_oldest_offset", + "Oldest Offset of a Broker at Topic/Partition", + List.of("topic", "partition"), + List.of(topicName, String.valueOf(partition)), + startOffset + ); + }); + state.description().partitions().forEach(p -> { + registry.gauge( + "kafka_topic_partition_in_sync_replica", + "Number of In-Sync Replicas for this Topic/Partition", + List.of("topic", "partition"), + List.of(topicName, String.valueOf(p.partition())), + p.isr().size() + ); + registry.gauge( + "kafka_topic_partition_replicas", + "Number of Replicas for this Topic/Partition", + List.of("topic", "partition"), + List.of(topicName, String.valueOf(p.partition())), + p.replicas().size() + ); + registry.gauge( + "kafka_topic_partition_leader", + "Leader Broker ID of this Topic/Partition (-1, if no leader)", + List.of("topic", "partition"), + List.of(topicName, String.valueOf(p.partition())), + Optional.ofNullable(p.leader()).map(Node::id).orElse(-1) + ); + }); + if (state.segmentStats() != null) { + registry.gauge( + "topic_bytes_disk", + "Disk size in bytes of a topic", + List.of("topic"), + List.of(topicName), + state.segmentStats().getSegmentSize() + ); + } + }); + } + + private static void fillConsumerGroupsMetrics(MetricsRegistry registry, ScrapedClusterState clusterState) { + registry.gauge( + "group_count", + "Number of consumer groups in the Kafka cluster", + List.of(), + List.of(), + clusterState.getConsumerGroupsStates().size() + ); + + clusterState.getConsumerGroupsStates().forEach((groupName, state) -> { + registry.gauge( + "group_state", + "State of the consumer group, value = ordinal of org.apache.kafka.common.ConsumerGroupState", + List.of("group"), + List.of(groupName), + state.description().state().ordinal() + ); + registry.gauge( + "group_member_count", + "Number of member assignments in the consumer group.", + List.of("group"), + List.of(groupName), + state.description().members().size() + ); + registry.gauge( + "group_host_count", + "Number of distinct hosts in the consumer group.", + List.of("group"), + List.of(groupName), + state.description().members().stream().map(MemberDescription::host).distinct().count() + ); + + state.committedOffsets().forEach((tp, committedOffset) -> { + registry.gauge( + "kafka_consumergroup_current_offset", + "Current Offset of a ConsumerGroup at Topic/Partition", + List.of("consumergroup", "topic", "partition"), + List.of(groupName, tp.topic(), String.valueOf(tp.partition())), + committedOffset + ); + + Optional.ofNullable(clusterState.getTopicStates().get(tp.topic())) + .flatMap(s -> Optional.ofNullable(s.endOffsets().get(tp.partition()))) + .ifPresent(endOffset -> + registry.gauge( + "kafka_consumergroup_lag", + "Current Approximate Lag of a ConsumerGroup at Topic/Partition", + List.of("consumergroup", "topic", "partition"), + List.of(groupName, tp.topic(), String.valueOf(tp.partition())), + endOffset - committedOffset //TODO: check +-1 + )); + + }); + }); + } +} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/JmxMetricsFormatter.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/jmx/JmxMetricsFormatter.java similarity index 95% rename from kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/JmxMetricsFormatter.java rename to kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/jmx/JmxMetricsFormatter.java index 4d3d31f50f4..7474cf9ad66 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/JmxMetricsFormatter.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/jmx/JmxMetricsFormatter.java @@ -1,5 +1,7 @@ -package com.provectus.kafka.ui.service.metrics; +package com.provectus.kafka.ui.service.metrics.scrape.jmx; +import com.provectus.kafka.ui.service.metrics.RawMetric; +import io.prometheus.client.Collector; import java.math.BigDecimal; import java.util.ArrayList; import java.util.LinkedHashMap; diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/JmxMetricsRetriever.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/jmx/JmxMetricsRetriever.java similarity index 58% rename from kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/JmxMetricsRetriever.java rename to kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/jmx/JmxMetricsRetriever.java index e7a58cbae27..40066a7340a 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/JmxMetricsRetriever.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/jmx/JmxMetricsRetriever.java @@ -1,6 +1,7 @@ -package com.provectus.kafka.ui.service.metrics; +package com.provectus.kafka.ui.service.metrics.scrape.jmx; -import com.provectus.kafka.ui.model.KafkaCluster; +import com.provectus.kafka.ui.model.MetricsScrapeProperties; +import com.provectus.kafka.ui.service.metrics.RawMetric; import java.io.Closeable; import java.util.ArrayList; import java.util.HashMap; @@ -17,15 +18,15 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.kafka.common.Node; +import org.springframework.stereotype.Component; import org.springframework.stereotype.Service; -import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.core.scheduler.Schedulers; -@Service +@Component //need to be a component, since @Slf4j -class JmxMetricsRetriever implements MetricsRetriever, Closeable { +public class JmxMetricsRetriever implements Closeable { private static final boolean SSL_JMX_SUPPORTED; @@ -43,35 +44,34 @@ public void close() { JmxSslSocketFactory.clearFactoriesCache(); } - @Override - public Flux retrieve(KafkaCluster c, Node node) { - if (isSslJmxEndpoint(c) && !SSL_JMX_SUPPORTED) { - log.warn("Cluster {} has jmx ssl configured, but it is not supported", c.getName()); - return Flux.empty(); + public Mono> retrieveFromNode(MetricsScrapeProperties scrapeProperties, Node node) { + if (isSslJmxEndpoint(scrapeProperties) && !SSL_JMX_SUPPORTED) { + log.warn("Cluster has jmx ssl configured, but it is not supported by app"); + return Mono.just(List.of()); } - return Mono.fromSupplier(() -> retrieveSync(c, node)) - .subscribeOn(Schedulers.boundedElastic()) - .flatMapMany(Flux::fromIterable); + return Mono.fromSupplier(() -> retrieveSync(scrapeProperties, node)) + .subscribeOn(Schedulers.boundedElastic()); } - private boolean isSslJmxEndpoint(KafkaCluster cluster) { - return cluster.getMetricsConfig().getKeystoreLocation() != null; + private boolean isSslJmxEndpoint(MetricsScrapeProperties scrapeProperties) { + return scrapeProperties.getKeystoreConfig() != null + && scrapeProperties.getKeystoreConfig().getKeystoreLocation() != null; } @SneakyThrows - private List retrieveSync(KafkaCluster c, Node node) { - String jmxUrl = JMX_URL + node.host() + ":" + c.getMetricsConfig().getPort() + "/" + JMX_SERVICE_TYPE; + private List retrieveSync(MetricsScrapeProperties scrapeProperties, Node node) { + String jmxUrl = JMX_URL + node.host() + ":" + scrapeProperties.getPort() + "/" + JMX_SERVICE_TYPE; log.debug("Collection JMX metrics for {}", jmxUrl); List result = new ArrayList<>(); - withJmxConnector(jmxUrl, c, jmxConnector -> getMetricsFromJmx(jmxConnector, result)); + withJmxConnector(jmxUrl, scrapeProperties, jmxConnector -> getMetricsFromJmx(jmxConnector, result)); log.debug("{} metrics collected for {}", result.size(), jmxUrl); return result; } private void withJmxConnector(String jmxUrl, - KafkaCluster c, + MetricsScrapeProperties scrapeProperties, Consumer consumer) { - var env = prepareJmxEnvAndSetThreadLocal(c); + var env = prepareJmxEnvAndSetThreadLocal(scrapeProperties); try (JMXConnector connector = JMXConnectorFactory.newJMXConnector(new JMXServiceURL(jmxUrl), env)) { try { connector.connect(env); @@ -87,25 +87,25 @@ private void withJmxConnector(String jmxUrl, } } - private Map prepareJmxEnvAndSetThreadLocal(KafkaCluster cluster) { - var metricsConfig = cluster.getMetricsConfig(); + private Map prepareJmxEnvAndSetThreadLocal(MetricsScrapeProperties scrapeProperties) { Map env = new HashMap<>(); - if (isSslJmxEndpoint(cluster)) { - var clusterSsl = cluster.getOriginalProperties().getSsl(); + if (isSslJmxEndpoint(scrapeProperties)) { + var truststoreConfig = scrapeProperties.getTruststoreConfig(); + var keystoreConfig = scrapeProperties.getKeystoreConfig(); JmxSslSocketFactory.setSslContextThreadLocal( - clusterSsl != null ? clusterSsl.getTruststoreLocation() : null, - clusterSsl != null ? clusterSsl.getTruststorePassword() : null, - metricsConfig.getKeystoreLocation(), - metricsConfig.getKeystorePassword() + truststoreConfig != null ? truststoreConfig.getTruststoreLocation() : null, + truststoreConfig != null ? truststoreConfig.getTruststorePassword() : null, + keystoreConfig != null ? keystoreConfig.getKeystoreLocation() : null, + keystoreConfig != null ? keystoreConfig.getKeystorePassword() : null ); JmxSslSocketFactory.editJmxConnectorEnv(env); } - if (StringUtils.isNotEmpty(metricsConfig.getUsername()) - && StringUtils.isNotEmpty(metricsConfig.getPassword())) { + if (StringUtils.isNotEmpty(scrapeProperties.getUsername()) + && StringUtils.isNotEmpty(scrapeProperties.getPassword())) { env.put( JMXConnector.CREDENTIALS, - new String[] {metricsConfig.getUsername(), metricsConfig.getPassword()} + new String[] {scrapeProperties.getUsername(), scrapeProperties.getPassword()} ); } return env; diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/jmx/JmxMetricsScraper.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/jmx/JmxMetricsScraper.java new file mode 100644 index 00000000000..4fc6dbf9111 --- /dev/null +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/jmx/JmxMetricsScraper.java @@ -0,0 +1,36 @@ +package com.provectus.kafka.ui.service.metrics.scrape.jmx; + +import static io.prometheus.client.Collector.MetricFamilySamples; + +import com.provectus.kafka.ui.model.MetricsScrapeProperties; +import com.provectus.kafka.ui.service.metrics.RawMetric; +import com.provectus.kafka.ui.service.metrics.scrape.PerBrokerScrapedMetrics; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import org.apache.kafka.common.Node; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import reactor.util.function.Tuples; + +public class JmxMetricsScraper { + + private final JmxMetricsRetriever jmxMetricsRetriever; + private final MetricsScrapeProperties scrapeProperties; + + public JmxMetricsScraper(MetricsScrapeProperties scrapeProperties, + JmxMetricsRetriever jmxMetricsRetriever) { + this.scrapeProperties = scrapeProperties; + this.jmxMetricsRetriever = jmxMetricsRetriever; + } + + public Mono scrape(Collection nodes) { + Mono>> collected = Flux.fromIterable(nodes) + .flatMap(n -> jmxMetricsRetriever.retrieveFromNode(scrapeProperties, n).map(metrics -> Tuples.of(n, metrics))) + .collectMap( + t -> t.getT1().id(), + t -> RawMetric.groupIntoMfs(t.getT2()).toList() + ); + return collected.map(PerBrokerScrapedMetrics::new); + } +} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/JmxSslSocketFactory.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/jmx/JmxSslSocketFactory.java similarity index 97% rename from kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/JmxSslSocketFactory.java rename to kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/jmx/JmxSslSocketFactory.java index fa84fc361c4..dd75538305a 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/JmxSslSocketFactory.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/jmx/JmxSslSocketFactory.java @@ -1,4 +1,4 @@ -package com.provectus.kafka.ui.service.metrics; +package com.provectus.kafka.ui.service.metrics.scrape.jmx; import com.google.common.base.Preconditions; import java.io.FileInputStream; @@ -61,9 +61,8 @@ class JmxSslSocketFactory extends javax.net.ssl.SSLSocketFactory { } catch (Exception e) { log.error("----------------------------------"); log.error("SSL can't be enabled for JMX retrieval. " - + "Make sure your java app run with '--add-opens java.rmi/javax.rmi.ssl=ALL-UNNAMED' arg. Err: {}", + + "Make sure your java app is running with '--add-opens java.rmi/javax.rmi.ssl=ALL-UNNAMED' arg. Err: {}", e.getMessage()); - log.trace("SSL can't be enabled for JMX retrieval", e); log.error("----------------------------------"); } SSL_JMX_SUPPORTED = sslJmxSupported; diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/prometheus/PrometheusEndpointParser.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/prometheus/PrometheusEndpointParser.java new file mode 100644 index 00000000000..f8794ac8eed --- /dev/null +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/prometheus/PrometheusEndpointParser.java @@ -0,0 +1,317 @@ +package com.provectus.kafka.ui.service.metrics.scrape.prometheus; + +import static io.prometheus.client.Collector.MetricFamilySamples.Sample; + +import com.google.common.base.Enums; +import io.prometheus.client.Collector.MetricFamilySamples; +import io.prometheus.client.Collector.Type; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Stream; + +public class PrometheusEndpointParser { + + // will be set if no TYPE provided (or it is unsupported) + private static final Type DEFAULT_TYPE = Type.GAUGE; + + private PrometheusEndpointParser() { + } + + private static class ParserContext { + final List registered = new ArrayList<>(); + + String name; + String help; + Type type; + String unit; + Set allowedNames = new HashSet<>(); + List samples = new ArrayList<>(); + + void registerAndReset() { + if (!samples.isEmpty()) { + registered.add( + new MetricFamilySamples( + name, + Optional.ofNullable(unit).orElse(""), + type, + Optional.ofNullable(help).orElse(name), + List.copyOf(samples)) + ); + } + //resetting state: + name = null; + help = null; + type = null; + unit = null; + allowedNames.clear(); + samples.clear(); + } + + List getRegistered() { + registerAndReset(); // last in progress metric should be registered + return registered; + } + } + + // general logic taken from https://github.com/prometheus/client_python/blob/master/prometheus_client/parser.py + public static List parse(Stream lines) { + ParserContext context = new ParserContext(); + lines.map(String::trim) + .filter(s -> !s.isBlank()) + .forEach(line -> { + if (line.charAt(0) == '#') { + String[] parts = line.split("[ \t]+", 4); + if (parts.length >= 3) { + switch (parts[1]) { + case "HELP" -> processHelp(context, parts); + case "TYPE" -> processType(context, parts); + case "UNIT" -> processUnit(context, parts); + default -> { /* probably a comment */ } + } + } + } else { + processSample(context, line); + } + }); + return context.getRegistered(); + } + + private static void processUnit(ParserContext context, String[] parts) { + if (!parts[2].equals(context.name)) { + // starting new metric family - need to register (if possible) prev one + context.registerAndReset(); + context.name = parts[2]; + context.type = DEFAULT_TYPE; + context.allowedNames.add(context.name); + } + if (parts.length == 4) { + context.unit = parts[3]; + } + } + + private static void processHelp(ParserContext context, String[] parts) { + if (!parts[2].equals(context.name)) { + // starting new metric family - need to register (if possible) prev one + context.registerAndReset(); + context.name = parts[2]; + context.type = DEFAULT_TYPE; + context.allowedNames.add(context.name); + } + if (parts.length == 4) { + context.help = unescapeHelp(parts[3]); + } + } + + private static void processType(ParserContext context, String[] parts) { + if (!parts[2].equals(context.name)) { + // starting new metric family - need to register (if possible) prev one + context.registerAndReset(); + context.name = parts[2]; + } + + context.type = Enums.getIfPresent(Type.class, parts[3].toUpperCase()).or(DEFAULT_TYPE); + switch (context.type) { + case SUMMARY -> { + context.allowedNames.add(context.name); + context.allowedNames.add(context.name + "_count"); + context.allowedNames.add(context.name + "_sum"); + context.allowedNames.add(context.name + "_created"); + } + case HISTOGRAM -> { + context.allowedNames.add(context.name + "_count"); + context.allowedNames.add(context.name + "_sum"); + context.allowedNames.add(context.name + "_bucket"); + context.allowedNames.add(context.name + "_created"); + } + case COUNTER -> { + context.allowedNames.add(context.name); + context.allowedNames.add(context.name + "_total"); + context.allowedNames.add(context.name + "_created"); + } + case INFO -> { + context.allowedNames.add(context.name); + context.allowedNames.add(context.name + "_info"); + } + default -> context.allowedNames.add(context.name); + } + } + + private static void processSample(ParserContext context, String line) { + parseSampleLine(line).ifPresent(sample -> { + if (!context.allowedNames.contains(sample.name)) { + // starting new metric family - need to register (if possible) prev one + context.registerAndReset(); + context.name = sample.name; + context.type = DEFAULT_TYPE; + context.allowedNames.add(sample.name); + } + context.samples.add(sample); + }); + } + + private static String unescapeHelp(String text) { + // algorithm from https://github.com/prometheus/client_python/blob/a2dae6caeaf3c300db416ba10a2a3271693addd4/prometheus_client/parser.py + if (text == null || !text.contains("\\")) { + return text; + } + StringBuilder result = new StringBuilder(); + boolean slash = false; + for (int c = 0; c < text.length(); c++) { + char charAt = text.charAt(c); + if (slash) { + if (charAt == '\\') { + result.append('\\'); + } else if (charAt == 'n') { + result.append('\n'); + } else { + result.append('\\').append(charAt); + } + slash = false; + } else { + if (charAt == '\\') { + slash = true; + } else { + result.append(charAt); + } + } + } + if (slash) { + result.append("\\"); + } + return result.toString(); + } + + //returns empty if line is not valid sample string + private static Optional parseSampleLine(String line) { + // algorithm copied from https://github.com/prometheus/client_python/blob/a2dae6caeaf3c300db416ba10a2a3271693addd4/prometheus_client/parser.py + StringBuilder name = new StringBuilder(); + StringBuilder labelname = new StringBuilder(); + StringBuilder labelvalue = new StringBuilder(); + StringBuilder value = new StringBuilder(); + List lblNames = new ArrayList<>(); + List lblVals = new ArrayList<>(); + + String state = "name"; + + for (int c = 0; c < line.length(); c++) { + char charAt = line.charAt(c); + if (state.equals("name")) { + if (charAt == '{') { + state = "startoflabelname"; + } else if (charAt == ' ' || charAt == '\t') { + state = "endofname"; + } else { + name.append(charAt); + } + } else if (state.equals("endofname")) { + if (charAt == ' ' || charAt == '\t') { + // do nothing + } else if (charAt == '{') { + state = "startoflabelname"; + } else { + value.append(charAt); + state = "value"; + } + } else if (state.equals("startoflabelname")) { + if (charAt == ' ' || charAt == '\t') { + // do nothing + } else if (charAt == '}') { + state = "endoflabels"; + } else { + labelname.append(charAt); + state = "labelname"; + } + } else if (state.equals("labelname")) { + if (charAt == '=') { + state = "labelvaluequote"; + } else if (charAt == '}') { + state = "endoflabels"; + } else if (charAt == ' ' || charAt == '\t') { + state = "labelvalueequals"; + } else { + labelname.append(charAt); + } + } else if (state.equals("labelvalueequals")) { + if (charAt == '=') { + state = "labelvaluequote"; + } else if (charAt == ' ' || charAt == '\t') { + // do nothing + } else { + return Optional.empty(); + } + } else if (state.equals("labelvaluequote")) { + if (charAt == '"') { + state = "labelvalue"; + } else if (charAt == ' ' || charAt == '\t') { + // do nothing + } else { + return Optional.empty(); + } + } else if (state.equals("labelvalue")) { + if (charAt == '\\') { + state = "labelvalueslash"; + } else if (charAt == '"') { + lblNames.add(labelname.toString()); + lblVals.add(labelvalue.toString()); + labelname.setLength(0); + labelvalue.setLength(0); + state = "nextlabel"; + } else { + labelvalue.append(charAt); + } + } else if (state.equals("labelvalueslash")) { + state = "labelvalue"; + if (charAt == '\\') { + labelvalue.append('\\'); + } else if (charAt == 'n') { + labelvalue.append('\n'); + } else if (charAt == '"') { + labelvalue.append('"'); + } else { + labelvalue.append('\\').append(charAt); + } + } else if (state.equals("nextlabel")) { + if (charAt == ',') { + state = "labelname"; + } else if (charAt == '}') { + state = "endoflabels"; + } else if (charAt == ' ' || charAt == '\t') { + // do nothing + } else { + return Optional.empty(); + } + } else if (state.equals("endoflabels")) { + if (charAt == ' ' || charAt == '\t') { + // do nothing + } else { + value.append(charAt); + state = "value"; + } + } else if (state.equals("value")) { + if (charAt == ' ' || charAt == '\t') { + break; // timestamps are NOT supported - ignoring + } else { + value.append(charAt); + } + } + } + return Optional.of(new Sample(name.toString(), lblNames, lblVals, parseDouble(value.toString()))); + } + + private static double parseDouble(String valueString) { + if (valueString.equalsIgnoreCase("NaN")) { + return Double.NaN; + } else if (valueString.equalsIgnoreCase("+Inf")) { + return Double.POSITIVE_INFINITY; + } else if (valueString.equalsIgnoreCase("-Inf")) { + return Double.NEGATIVE_INFINITY; + } + return Double.parseDouble(valueString); + } + + +} + diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/prometheus/PrometheusMetricsRetriever.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/prometheus/PrometheusMetricsRetriever.java new file mode 100644 index 00000000000..d7ad5b3739f --- /dev/null +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/prometheus/PrometheusMetricsRetriever.java @@ -0,0 +1,54 @@ +package com.provectus.kafka.ui.service.metrics.scrape.prometheus; + +import static io.prometheus.client.Collector.MetricFamilySamples; + +import com.provectus.kafka.ui.model.MetricsScrapeProperties; +import com.provectus.kafka.ui.util.WebClientConfigurator; +import java.util.List; +import java.util.Optional; +import lombok.extern.slf4j.Slf4j; +import org.springframework.util.unit.DataSize; +import org.springframework.web.reactive.function.client.WebClient; +import org.springframework.web.util.UriComponentsBuilder; +import reactor.core.publisher.Mono; + +@Slf4j +class PrometheusMetricsRetriever { + + private static final String METRICS_ENDPOINT_PATH = "/metrics"; + private static final int DEFAULT_EXPORTER_PORT = 11001; + + private final int port; + private final boolean sslEnabled; + private final WebClient webClient; + + PrometheusMetricsRetriever(MetricsScrapeProperties scrapeProperties) { + this.port = Optional.ofNullable(scrapeProperties.getPort()).orElse(DEFAULT_EXPORTER_PORT); + this.sslEnabled = scrapeProperties.isSsl() || scrapeProperties.getKeystoreConfig() != null; + this.webClient = new WebClientConfigurator() + .configureBufferSize(DataSize.ofMegabytes(20)) + .configureBasicAuth(scrapeProperties.getUsername(), scrapeProperties.getPassword()) + .configureSsl(scrapeProperties.getTruststoreConfig(), scrapeProperties.getKeystoreConfig()) + .build(); + } + + Mono> retrieve(String host) { + log.debug("Retrieving metrics from prometheus endpoint: {}:{}", host, port); + + var uri = UriComponentsBuilder.newInstance() + .scheme(sslEnabled ? "https" : "http") + .host(host) + .port(port) + .path(METRICS_ENDPOINT_PATH) + .build() + .toUri(); + + return webClient.get() + .uri(uri) + .retrieve() + .bodyToMono(String.class) + .doOnError(e -> log.error("Error while getting metrics from {}", host, e)) + .map(body -> PrometheusEndpointParser.parse(body.lines())) + .onErrorResume(th -> Mono.just(List.of())); + } +} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/prometheus/PrometheusScraper.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/prometheus/PrometheusScraper.java new file mode 100644 index 00000000000..1d5f5e3fba6 --- /dev/null +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/prometheus/PrometheusScraper.java @@ -0,0 +1,30 @@ +package com.provectus.kafka.ui.service.metrics.scrape.prometheus; + +import static io.prometheus.client.Collector.MetricFamilySamples; + +import com.provectus.kafka.ui.model.MetricsScrapeProperties; +import com.provectus.kafka.ui.service.metrics.scrape.PerBrokerScrapedMetrics; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import org.apache.kafka.common.Node; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import reactor.util.function.Tuple2; +import reactor.util.function.Tuples; + +public class PrometheusScraper { + + private final PrometheusMetricsRetriever retriever; + + public PrometheusScraper(MetricsScrapeProperties scrapeProperties) { + this.retriever = new PrometheusMetricsRetriever(scrapeProperties); + } + + public Mono scrape(Collection clusterNodes) { + Mono>> collected = Flux.fromIterable(clusterNodes) + .flatMap(n -> retriever.retrieve(n.host()).map(metrics -> Tuples.of(n, metrics))) + .collectMap(t -> t.getT1().id(), Tuple2::getT2); + return collected.map(PerBrokerScrapedMetrics::new); + } +} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/sink/KafkaSink.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/sink/KafkaSink.java new file mode 100644 index 00000000000..d88973bbea3 --- /dev/null +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/sink/KafkaSink.java @@ -0,0 +1,73 @@ +package com.provectus.kafka.ui.service.metrics.sink; + +import static com.provectus.kafka.ui.service.MessagesService.createProducer; +import static com.provectus.kafka.ui.service.metrics.prometheus.PrometheusExpose.escapedLabelValue; +import static io.prometheus.client.Collector.MetricFamilySamples; +import static io.prometheus.client.Collector.doubleToGoString; +import static org.apache.kafka.clients.producer.ProducerConfig.COMPRESSION_TYPE_CONFIG; + +import com.fasterxml.jackson.databind.json.JsonMapper; +import com.provectus.kafka.ui.config.ClustersProperties; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoUnit; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.stream.Stream; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.clients.producer.ProducerRecord; +import reactor.core.publisher.Mono; + +/* + * Format of records copied from https://github.com/Telefonica/prometheus-kafka-adapter + */ +@RequiredArgsConstructor +class KafkaSink implements MetricsSink { + + record KafkaMetric(String timestamp, String value, String name, Map labels) { } + + private static final JsonMapper JSON_MAPPER = new JsonMapper(); + + private static final Map PRODUCER_ADDITIONAL_CONFIGS = Map.of(COMPRESSION_TYPE_CONFIG, "gzip"); + + private final String topic; + private final Producer producer; + + static KafkaSink create(ClustersProperties.Cluster cluster, String targetTopic) { + return new KafkaSink(targetTopic, createProducer(cluster, PRODUCER_ADDITIONAL_CONFIGS)); + } + + @Override + public Mono send(Stream metrics) { + return Mono.fromRunnable(() -> { + String ts = Instant.now() + .truncatedTo(ChronoUnit.SECONDS) + .atZone(ZoneOffset.UTC) + .format(DateTimeFormatter.ISO_DATE_TIME); + + metrics.flatMap(m -> createRecord(ts, m)).forEach(producer::send); + }); + } + + private Stream> createRecord(String ts, MetricFamilySamples metrics) { + return metrics.samples.stream() + .map(sample -> { + var lbls = new LinkedHashMap(); + lbls.put("__name__", sample.name); + for (int i = 0; i < sample.labelNames.size(); i++) { + lbls.put(sample.labelNames.get(i), escapedLabelValue(sample.labelValues.get(i))); + } + var km = new KafkaMetric(ts, doubleToGoString(sample.value), sample.name, lbls); + return new ProducerRecord<>(topic, toJsonBytes(km)); + }); + } + + @SneakyThrows + private static byte[] toJsonBytes(KafkaMetric m) { + return JSON_MAPPER.writeValueAsBytes(m); + } + +} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/sink/MetricsSink.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/sink/MetricsSink.java new file mode 100644 index 00000000000..0229490c05e --- /dev/null +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/sink/MetricsSink.java @@ -0,0 +1,56 @@ +package com.provectus.kafka.ui.service.metrics.sink; + +import static io.prometheus.client.Collector.MetricFamilySamples; +import static org.springframework.util.StringUtils.hasText; + +import com.provectus.kafka.ui.config.ClustersProperties; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.stream.Stream; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +public interface MetricsSink { + + static MetricsSink create(ClustersProperties.Cluster cluster) { + List sinks = new ArrayList<>(); + Optional.ofNullable(cluster.getMetrics()) + .flatMap(metrics -> Optional.ofNullable(metrics.getStore())) + .flatMap(store -> Optional.ofNullable(store.getPrometheus())) + .ifPresent(prometheusConf -> { + if (hasText(prometheusConf.getUrl()) && Boolean.TRUE.equals(prometheusConf.getRemoteWrite())) { + sinks.add(new PrometheusRemoteWriteSink(prometheusConf.getUrl(), cluster.getSsl())); + } + if (hasText(prometheusConf.getPushGatewayUrl())) { + sinks.add( + PrometheusPushGatewaySink.create( + prometheusConf.getPushGatewayUrl(), + prometheusConf.getPushGatewayJobName(), + prometheusConf.getPushGatewayUsername(), + prometheusConf.getPushGatewayPassword() + )); + } + }); + + Optional.ofNullable(cluster.getMetrics()) + .flatMap(metrics -> Optional.ofNullable(metrics.getStore())) + .flatMap(store -> Optional.ofNullable(store.getKafka())) + .flatMap(kafka -> Optional.ofNullable(kafka.getTopic())) + .ifPresent(topic -> sinks.add(KafkaSink.create(cluster, topic))); + + return compoundSink(sinks); + } + + private static MetricsSink compoundSink(List sinks) { + return metricsStream -> { + var materialized = metricsStream.toList(); + return Flux.fromIterable(sinks) + .flatMap(sink -> sink.send(materialized.stream())) + .then(); + }; + } + + Mono send(Stream metrics); + +} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/sink/PrometheusPushGatewaySink.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/sink/PrometheusPushGatewaySink.java new file mode 100644 index 00000000000..60a2bf5ca22 --- /dev/null +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/sink/PrometheusPushGatewaySink.java @@ -0,0 +1,62 @@ +package com.provectus.kafka.ui.service.metrics.sink; + +import static io.prometheus.client.Collector.MetricFamilySamples; +import static org.springframework.util.StringUtils.hasText; + +import io.prometheus.client.Collector; +import io.prometheus.client.exporter.BasicAuthHttpConnectionFactory; +import io.prometheus.client.exporter.PushGateway; +import jakarta.annotation.Nullable; +import java.net.URL; +import java.util.List; +import java.util.Optional; +import java.util.stream.Stream; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; + +@RequiredArgsConstructor +class PrometheusPushGatewaySink implements MetricsSink { + + private static final String DEFAULT_PGW_JOB_NAME = "kafkaui"; + + private final PushGateway pushGateway; + private final String job; + + @SneakyThrows + static PrometheusPushGatewaySink create(String url, + @Nullable String jobName, + @Nullable String username, + @Nullable String passw) { + var pushGateway = new PushGateway(new URL(url)); + if (hasText(username) && hasText(passw)) { + pushGateway.setConnectionFactory(new BasicAuthHttpConnectionFactory(username, passw)); + } + return new PrometheusPushGatewaySink( + pushGateway, + Optional.ofNullable(jobName).orElse(DEFAULT_PGW_JOB_NAME) + ); + } + + @Override + public Mono send(Stream metrics) { + List metricsToPush = metrics.toList(); + if (metricsToPush.isEmpty()) { + return Mono.empty(); + } + return Mono.fromRunnable(() -> pushSync(metricsToPush)) + .subscribeOn(Schedulers.boundedElastic()); + } + + @SneakyThrows + private void pushSync(List metricsToPush) { + Collector allMetrics = new Collector() { + @Override + public List collect() { + return metricsToPush; + } + }; + pushGateway.push(allMetrics, job); + } +} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/sink/PrometheusRemoteWriteSink.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/sink/PrometheusRemoteWriteSink.java new file mode 100644 index 00000000000..55f01272358 --- /dev/null +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/sink/PrometheusRemoteWriteSink.java @@ -0,0 +1,79 @@ +package com.provectus.kafka.ui.service.metrics.sink; + +import static io.prometheus.client.Collector.MetricFamilySamples; +import static prometheus.Types.Label; +import static prometheus.Types.Sample; +import static prometheus.Types.TimeSeries; + +import com.provectus.kafka.ui.config.ClustersProperties.TruststoreConfig; +import com.provectus.kafka.ui.service.metrics.prometheus.PrometheusExpose; +import com.provectus.kafka.ui.util.WebClientConfigurator; +import jakarta.annotation.Nullable; +import java.net.URI; +import java.util.stream.Stream; +import lombok.SneakyThrows; +import org.springframework.util.unit.DataSize; +import org.springframework.web.reactive.function.client.WebClient; +import org.xerial.snappy.Snappy; +import prometheus.Remote; +import reactor.core.publisher.Mono; + +class PrometheusRemoteWriteSink implements MetricsSink { + + private final URI writeEndpoint; + private final WebClient webClient; + + PrometheusRemoteWriteSink(String prometheusUrl, @Nullable TruststoreConfig truststoreConfig) { + this.writeEndpoint = URI.create(prometheusUrl).resolve("/api/v1/write"); + this.webClient = new WebClientConfigurator() + .configureSsl(truststoreConfig, null) + .configureBufferSize(DataSize.ofMegabytes(20)) + .build(); + } + + @SneakyThrows + @Override + public Mono send(Stream metrics) { + byte[] bytesToWrite = Snappy.compress(createWriteRequest(metrics).toByteArray()); + return webClient.post() + .uri(writeEndpoint) + .header("Content-Type", "application/x-protobuf") + .header("User-Agent", "promremote-kui/0.1.0") + .header("Content-Encoding", "snappy") + .header("X-Prometheus-Remote-Write-Version", "0.1.0") + .bodyValue(bytesToWrite) + .retrieve() + .toBodilessEntity() + .then(); + } + + private static Remote.WriteRequest createWriteRequest(Stream metrics) { + long currentTs = System.currentTimeMillis(); + Remote.WriteRequest.Builder request = Remote.WriteRequest.newBuilder(); + metrics.forEach(mfs -> { + for (MetricFamilySamples.Sample sample : mfs.samples) { + TimeSeries.Builder timeSeriesBuilder = TimeSeries.newBuilder(); + timeSeriesBuilder.addLabels( + Label.newBuilder().setName("__name__").setValue(sample.name) + ); + for (int i = 0; i < sample.labelNames.size(); i++) { + timeSeriesBuilder.addLabels( + Label.newBuilder() + .setName(sample.labelNames.get(i)) + .setValue(PrometheusExpose.escapedLabelValue(sample.labelValues.get(i))) + ); + } + timeSeriesBuilder.addSamples( + Sample.newBuilder() + .setValue(sample.value) + .setTimestamp(currentTs) + ); + request.addTimeseries(timeSeriesBuilder); + } + }); + //TODO: pass Metadata + return request.build(); + } + + +} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/KafkaServicesValidation.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/KafkaServicesValidation.java index 4b8af81f851..e1b725aa34a 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/KafkaServicesValidation.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/KafkaServicesValidation.java @@ -19,6 +19,7 @@ import org.apache.kafka.clients.admin.AdminClient; import org.apache.kafka.clients.admin.AdminClientConfig; import org.springframework.util.ResourceUtils; +import prometheus.query.api.PrometheusClientApi; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -46,7 +47,7 @@ private static Mono invalid(Throwable th) { public static Optional validateTruststore(TruststoreConfig truststoreConfig) { if (truststoreConfig.getTruststoreLocation() != null && truststoreConfig.getTruststorePassword() != null) { try (FileInputStream fileInputStream = new FileInputStream( - (ResourceUtils.getFile(truststoreConfig.getTruststoreLocation())))) { + (ResourceUtils.getFile(truststoreConfig.getTruststoreLocation())))) { KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType()); trustStore.load(fileInputStream, truststoreConfig.getTruststorePassword().toCharArray()); TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance( @@ -141,5 +142,18 @@ public static Mono validateKsql( .onErrorResume(KafkaServicesValidation::invalid); } + public static Mono validatePrometheusStore( + Supplier> clientSupplier) { + ReactiveFailover client; + try { + client = clientSupplier.get(); + } catch (Exception e) { + log.error("Error creating Prometheus client", e); + return invalid("Error creating Prometheus client: " + e.getMessage()); + } + return client.mono(c -> c.query("1", null, null)) + .then(valid()) + .onErrorResume(KafkaServicesValidation::invalid); + } } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/ReactiveFailover.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/ReactiveFailover.java index 0293e4f9250..373a85d8bf8 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/ReactiveFailover.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/ReactiveFailover.java @@ -81,9 +81,6 @@ private Mono mono(Function> f, List> candid .flatMap(f) .onErrorResume(failoverExceptionsPredicate, th -> { publisher.markFailed(); - if (candidates.size() == 1) { - return Mono.error(th); - } var newCandidates = candidates.stream().skip(1).filter(PublisherHolder::isActive).toList(); if (newCandidates.isEmpty()) { return Mono.error(th); @@ -106,9 +103,6 @@ private Flux flux(Function> f, List> candid .flatMapMany(f) .onErrorResume(failoverExceptionsPredicate, th -> { publisher.markFailed(); - if (candidates.size() == 1) { - return Flux.error(th); - } var newCandidates = candidates.stream().skip(1).filter(PublisherHolder::isActive).toList(); if (newCandidates.isEmpty()) { return Flux.error(th); diff --git a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/container/PrometheusContainer.java b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/container/PrometheusContainer.java new file mode 100644 index 00000000000..9e026c69816 --- /dev/null +++ b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/container/PrometheusContainer.java @@ -0,0 +1,19 @@ +package com.provectus.kafka.ui.container; + +import org.testcontainers.containers.GenericContainer; + +public class PrometheusContainer extends GenericContainer { + + public PrometheusContainer() { + super("prom/prometheus:latest"); + setCommandParts(new String[] { + "--web.enable-remote-write-receiver", + "--config.file=/etc/prometheus/prometheus.yml" + }); + addExposedPort(9090); + } + + public String url() { + return "http://" + getHost() + ":" + getMappedPort(9090); + } +} diff --git a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/model/PartitionDistributionStatsTest.java b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/model/PartitionDistributionStatsTest.java index c83c4f5cd86..6129b065d94 100644 --- a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/model/PartitionDistributionStatsTest.java +++ b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/model/PartitionDistributionStatsTest.java @@ -23,28 +23,23 @@ void skewCalculatedBasedOnPartitionsCounts() { Node n4 = new Node(4, "n4", 9092); var stats = PartitionDistributionStats.create( - Statistics.builder() - .clusterDescription( - new ReactiveAdminClient.ClusterDescription(null, "test", Set.of(n1, n2, n3), null)) - .topicDescriptions( - Map.of( - "t1", new TopicDescription( - "t1", false, - List.of( - new TopicPartitionInfo(0, n1, List.of(n1, n2), List.of(n1, n2)), - new TopicPartitionInfo(1, n2, List.of(n2, n3), List.of(n2, n3)) - ) - ), - "t2", new TopicDescription( - "t2", false, - List.of( - new TopicPartitionInfo(0, n1, List.of(n1, n2), List.of(n1, n2)), - new TopicPartitionInfo(1, null, List.of(n2, n1), List.of(n1)) - ) - ) + List.of( + new TopicDescription( + "t1", false, + List.of( + new TopicPartitionInfo(0, n1, List.of(n1, n2), List.of(n1, n2)), + new TopicPartitionInfo(1, n2, List.of(n2, n3), List.of(n2, n3)) + ) + ), + new TopicDescription( + "t2", false, + List.of( + new TopicPartitionInfo(0, n1, List.of(n1, n2), List.of(n1, n2)), + new TopicPartitionInfo(1, null, List.of(n2, n1), List.of(n1)) ) ) - .build(), 4 + ), + 4 ); assertThat(stats.getPartitionLeaders()) diff --git a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/TopicsServicePaginationTest.java b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/TopicsServicePaginationTest.java index 3a6cebc8347..fd2059752f3 100644 --- a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/TopicsServicePaginationTest.java +++ b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/TopicsServicePaginationTest.java @@ -71,7 +71,7 @@ public void shouldListFirst25Topics() { .map(Objects::toString) .map(name -> new TopicDescription(name, false, List.of())) .map(topicDescription -> InternalTopic.from(topicDescription, List.of(), null, - Metrics.empty(), InternalLogDirStats.empty(), "_")) + Metrics.empty(), null, null, "_")) .collect(Collectors.toMap(InternalTopic::getName, Function.identity())) ); @@ -97,7 +97,7 @@ public void shouldListFirst25TopicsSortedByNameDescendingOrder() { .map(Objects::toString) .map(name -> new TopicDescription(name, false, List.of())) .map(topicDescription -> InternalTopic.from(topicDescription, List.of(), null, - Metrics.empty(), InternalLogDirStats.empty(), "_")) + Metrics.empty(), null, null, "_")) .collect(Collectors.toMap(InternalTopic::getName, Function.identity())); init(internalTopics); @@ -124,7 +124,7 @@ public void shouldCalculateCorrectPageCountForNonDivisiblePageSize() { .map(Objects::toString) .map(name -> new TopicDescription(name, false, List.of())) .map(topicDescription -> InternalTopic.from(topicDescription, List.of(), null, - Metrics.empty(), InternalLogDirStats.empty(), "_")) + Metrics.empty(), null, null, "_")) .collect(Collectors.toMap(InternalTopic::getName, Function.identity())) ); @@ -143,7 +143,7 @@ public void shouldCorrectlyHandleNonPositivePageNumberAndPageSize() { .map(Objects::toString) .map(name -> new TopicDescription(name, false, List.of())) .map(topicDescription -> InternalTopic.from(topicDescription, List.of(), null, - Metrics.empty(), InternalLogDirStats.empty(), "_")) + Metrics.empty(), null, null, "_")) .collect(Collectors.toMap(InternalTopic::getName, Function.identity())) ); @@ -162,7 +162,7 @@ public void shouldListBotInternalAndNonInternalTopics() { .map(Objects::toString) .map(name -> new TopicDescription(name, Integer.parseInt(name) % 10 == 0, List.of())) .map(topicDescription -> InternalTopic.from(topicDescription, List.of(), null, - Metrics.empty(), InternalLogDirStats.empty(), "_")) + Metrics.empty(), null, null, "_")) .collect(Collectors.toMap(InternalTopic::getName, Function.identity())) ); @@ -183,7 +183,7 @@ public void shouldListOnlyNonInternalTopics() { .map(Objects::toString) .map(name -> new TopicDescription(name, Integer.parseInt(name) % 5 == 0, List.of())) .map(topicDescription -> InternalTopic.from(topicDescription, List.of(), null, - Metrics.empty(), InternalLogDirStats.empty(), "_")) + Metrics.empty(), null, null, "_")) .collect(Collectors.toMap(InternalTopic::getName, Function.identity())) ); @@ -204,7 +204,7 @@ public void shouldListOnlyTopicsContainingOne() { .map(Objects::toString) .map(name -> new TopicDescription(name, false, List.of())) .map(topicDescription -> InternalTopic.from(topicDescription, List.of(), null, - Metrics.empty(), InternalLogDirStats.empty(), "_")) + Metrics.empty(), null, null, "_")) .collect(Collectors.toMap(InternalTopic::getName, Function.identity())) ); @@ -226,7 +226,7 @@ public void shouldListTopicsOrderedByPartitionsCount() { new TopicPartitionInfo(p, null, List.of(), List.of())) .collect(Collectors.toList()))) .map(topicDescription -> InternalTopic.from(topicDescription, List.of(), InternalPartitionsOffsets.empty(), - Metrics.empty(), InternalLogDirStats.empty(), "_")) + Metrics.empty(), null, null, "_")) .collect(Collectors.toMap(InternalTopic::getName, Function.identity())); init(internalTopics); diff --git a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/integration/odd/TopicsExporterTest.java b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/integration/odd/TopicsExporterTest.java index cb4103467be..26d98865cf2 100644 --- a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/integration/odd/TopicsExporterTest.java +++ b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/integration/odd/TopicsExporterTest.java @@ -1,5 +1,7 @@ package com.provectus.kafka.ui.service.integration.odd; +import static com.provectus.kafka.ui.service.metrics.scrape.ScrapedClusterState.TopicState; +import static com.provectus.kafka.ui.service.metrics.scrape.ScrapedClusterState.empty; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyString; @@ -9,6 +11,7 @@ import com.provectus.kafka.ui.model.KafkaCluster; import com.provectus.kafka.ui.model.Statistics; import com.provectus.kafka.ui.service.StatisticsCache; +import com.provectus.kafka.ui.service.metrics.scrape.ScrapedClusterState; import com.provectus.kafka.ui.sr.api.KafkaSrClientApi; import com.provectus.kafka.ui.sr.model.SchemaSubject; import com.provectus.kafka.ui.sr.model.SchemaType; @@ -59,15 +62,22 @@ void doesNotExportTopicsWhichDontFitFiltrationRule() { .thenReturn(Mono.error(WebClientResponseException.create(404, "NF", new HttpHeaders(), null, null, null))); stats = Statistics.empty() .toBuilder() - .topicDescriptions( - Map.of( - "_hidden", new TopicDescription("_hidden", false, List.of( - new TopicPartitionInfo(0, null, List.of(), List.of()) - )), - "visible", new TopicDescription("visible", false, List.of( - new TopicPartitionInfo(0, null, List.of(), List.of()) - )) - ) + .clusterState( + empty().toBuilder().topicStates( + Map.of( + "_hidden", + new TopicState( + "_hidden", + new TopicDescription("_hidden", false, List.of( + new TopicPartitionInfo(0, null, List.of(), List.of()) + )), null, null, null, null, null), + "visible", + new TopicState("visible", + new TopicDescription("visible", false, List.of( + new TopicPartitionInfo(0, null, List.of(), List.of()) + )), null, null, null, null, null) + ) + ).build() ) .build(); @@ -101,40 +111,44 @@ void doesExportTopicData() { stats = Statistics.empty() .toBuilder() - .topicDescriptions( - Map.of( - "testTopic", - new TopicDescription( - "testTopic", - false, - List.of( - new TopicPartitionInfo( - 0, - null, + .clusterState( + ScrapedClusterState.empty().toBuilder() + .topicStates( + Map.of( + "testTopic", + new TopicState( + "testTopic", + new TopicDescription( + "testTopic", + false, + List.of( + new TopicPartitionInfo( + 0, + null, + List.of( + new Node(1, "host1", 9092), + new Node(2, "host2", 9092) + ), + List.of()) + ) + ), List.of( - new Node(1, "host1", 9092), - new Node(2, "host2", 9092) + new ConfigEntry( + "custom.config", + "100500", + ConfigEntry.ConfigSource.DYNAMIC_TOPIC_CONFIG, + false, + false, + List.of(), + ConfigEntry.ConfigType.INT, + null + ) ), - List.of()) - )) - ) - ) - .topicConfigs( - Map.of( - "testTopic", List.of( - new ConfigEntry( - "custom.config", - "100500", - ConfigEntry.ConfigSource.DYNAMIC_TOPIC_CONFIG, - false, - false, - List.of(), - ConfigEntry.ConfigType.INT, - null + null, null, null, null + ) ) ) - ) - ) + .build()) .build(); StepVerifier.create(topicsExporter.export(cluster)) diff --git a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/ksql/KsqlApiClientTest.java b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/ksql/KsqlApiClientTest.java index f266e07c6d5..2bca09a1c78 100644 --- a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/ksql/KsqlApiClientTest.java +++ b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/ksql/KsqlApiClientTest.java @@ -11,12 +11,14 @@ import java.math.BigDecimal; import java.time.Duration; import java.util.Map; +import org.junit.Ignore; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.testcontainers.shaded.org.awaitility.Awaitility; import reactor.test.StepVerifier; +@Ignore class KsqlApiClientTest extends AbstractIntegrationTest { @BeforeAll diff --git a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/metrics/PrometheusEndpointMetricsParserTest.java b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/metrics/PrometheusEndpointMetricsParserTest.java deleted file mode 100644 index 294215c8b18..00000000000 --- a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/metrics/PrometheusEndpointMetricsParserTest.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.provectus.kafka.ui.service.metrics; - -import static org.assertj.core.api.Assertions.assertThat; - -import java.util.Map; -import java.util.Optional; -import org.junit.jupiter.api.Test; - -class PrometheusEndpointMetricsParserTest { - - @Test - void test() { - String metricsString = - "kafka_server_BrokerTopicMetrics_FifteenMinuteRate" - + "{name=\"BytesOutPerSec\",topic=\"__confluent.support.metrics\",} 123.1234"; - - Optional parsedOpt = PrometheusEndpointMetricsParser.parse(metricsString); - - assertThat(parsedOpt).hasValueSatisfying(metric -> { - assertThat(metric.name()).isEqualTo("kafka_server_BrokerTopicMetrics_FifteenMinuteRate"); - assertThat(metric.value()).isEqualTo("123.1234"); - assertThat(metric.labels()).containsExactlyEntriesOf( - Map.of( - "name", "BytesOutPerSec", - "topic", "__confluent.support.metrics" - )); - }); - } - -} \ No newline at end of file diff --git a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/metrics/PrometheusMetricsRetrieverTest.java b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/metrics/PrometheusMetricsRetrieverTest.java deleted file mode 100644 index 9cc0494039d..00000000000 --- a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/metrics/PrometheusMetricsRetrieverTest.java +++ /dev/null @@ -1,97 +0,0 @@ -package com.provectus.kafka.ui.service.metrics; - -import com.provectus.kafka.ui.model.MetricsConfig; -import java.io.IOException; -import java.math.BigDecimal; -import java.util.List; -import java.util.Map; -import okhttp3.mockwebserver.MockResponse; -import okhttp3.mockwebserver.MockWebServer; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.springframework.web.reactive.function.client.WebClient; -import reactor.test.StepVerifier; - -class PrometheusMetricsRetrieverTest { - - private final PrometheusMetricsRetriever retriever = new PrometheusMetricsRetriever(); - - private final MockWebServer mockWebServer = new MockWebServer(); - - @BeforeEach - void startMockServer() throws IOException { - mockWebServer.start(); - } - - @AfterEach - void stopMockServer() throws IOException { - mockWebServer.close(); - } - - @Test - void callsMetricsEndpointAndConvertsResponceToRawMetric() { - var url = mockWebServer.url("/metrics"); - mockWebServer.enqueue(prepareResponse()); - - MetricsConfig metricsConfig = prepareMetricsConfig(url.port(), null, null); - - StepVerifier.create(retriever.retrieve(WebClient.create(), url.host(), metricsConfig)) - .expectNextSequence(expectedRawMetrics()) - // third metric should not be present, since it has "NaN" value - .verifyComplete(); - } - - @Test - void callsSecureMetricsEndpointAndConvertsResponceToRawMetric() { - var url = mockWebServer.url("/metrics"); - mockWebServer.enqueue(prepareResponse()); - - - MetricsConfig metricsConfig = prepareMetricsConfig(url.port(), "username", "password"); - - StepVerifier.create(retriever.retrieve(WebClient.create(), url.host(), metricsConfig)) - .expectNextSequence(expectedRawMetrics()) - // third metric should not be present, since it has "NaN" value - .verifyComplete(); - } - - MockResponse prepareResponse() { - // body copied from real jmx exporter - return new MockResponse().setBody( - "# HELP kafka_server_KafkaRequestHandlerPool_FifteenMinuteRate Attribute exposed for management \n" - + "# TYPE kafka_server_KafkaRequestHandlerPool_FifteenMinuteRate untyped\n" - + "kafka_server_KafkaRequestHandlerPool_FifteenMinuteRate{name=\"RequestHandlerAvgIdlePercent\",} 0.898\n" - + "# HELP kafka_server_socket_server_metrics_request_size_avg The average size of requests sent. \n" - + "# TYPE kafka_server_socket_server_metrics_request_size_avg untyped\n" - + "kafka_server_socket_server_metrics_request_size_avg{listener=\"PLAIN\",networkProcessor=\"1\",} 101.1\n" - + "kafka_server_socket_server_metrics_request_size_avg{listener=\"PLAIN2\",networkProcessor=\"5\",} NaN" - ); - } - - MetricsConfig prepareMetricsConfig(Integer port, String username, String password) { - return MetricsConfig.builder() - .ssl(false) - .port(port) - .type(MetricsConfig.PROMETHEUS_METRICS_TYPE) - .username(username) - .password(password) - .build(); - } - - List expectedRawMetrics() { - - var firstMetric = RawMetric.create( - "kafka_server_KafkaRequestHandlerPool_FifteenMinuteRate", - Map.of("name", "RequestHandlerAvgIdlePercent"), - new BigDecimal("0.898") - ); - - var secondMetric = RawMetric.create( - "kafka_server_socket_server_metrics_request_size_avg", - Map.of("listener", "PLAIN", "networkProcessor", "1"), - new BigDecimal("101.1") - ); - return List.of(firstMetric, secondMetric); - } -} diff --git a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/metrics/WellKnownMetricsTest.java b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/metrics/WellKnownMetricsTest.java deleted file mode 100644 index c1c9c04058a..00000000000 --- a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/metrics/WellKnownMetricsTest.java +++ /dev/null @@ -1,93 +0,0 @@ -package com.provectus.kafka.ui.service.metrics; - -import static org.assertj.core.api.Assertions.assertThat; - -import com.provectus.kafka.ui.model.Metrics; -import java.math.BigDecimal; -import java.util.Arrays; -import java.util.Map; -import java.util.Optional; -import org.apache.kafka.common.Node; -import org.junit.jupiter.api.Test; - -class WellKnownMetricsTest { - - private final WellKnownMetrics wellKnownMetrics = new WellKnownMetrics(); - - @Test - void bytesIoTopicMetricsPopulated() { - populateWith( - new Node(0, "host", 123), - "kafka_server_BrokerTopicMetrics_FifteenMinuteRate{name=\"BytesInPerSec\",topic=\"test-topic\",} 1.0", - "kafka_server_BrokerTopicMetrics_FifteenMinuteRate{name=\"BytesOutPerSec\",topic=\"test-topic\",} 2.0", - "kafka_server_brokertopicmetrics_fifteenminuterate{name=\"bytesinpersec\",topic=\"test-topic\",} 1.0", - "kafka_server_brokertopicmetrics_fifteenminuterate{name=\"bytesoutpersec\",topic=\"test-topic\",} 2.0", - "some_unknown_prefix_brokertopicmetrics_fifteenminuterate{name=\"bytesinpersec\",topic=\"test-topic\",} 1.0", - "some_unknown_prefix_brokertopicmetrics_fifteenminuterate{name=\"bytesoutpersec\",topic=\"test-topic\",} 2.0" - ); - assertThat(wellKnownMetrics.bytesInFifteenMinuteRate) - .containsEntry("test-topic", new BigDecimal("3.0")); - assertThat(wellKnownMetrics.bytesOutFifteenMinuteRate) - .containsEntry("test-topic", new BigDecimal("6.0")); - } - - @Test - void bytesIoBrokerMetricsPopulated() { - populateWith( - new Node(1, "host1", 123), - "kafka_server_BrokerTopicMetrics_FifteenMinuteRate{name=\"BytesInPerSec\",} 1.0", - "kafka_server_BrokerTopicMetrics_FifteenMinuteRate{name=\"BytesOutPerSec\",} 2.0" - ); - populateWith( - new Node(2, "host2", 345), - "some_unknown_prefix_brokertopicmetrics_fifteenminuterate{name=\"bytesinpersec\",} 10.0", - "some_unknown_prefix_brokertopicmetrics_fifteenminuterate{name=\"bytesoutpersec\",} 20.0" - ); - - assertThat(wellKnownMetrics.brokerBytesInFifteenMinuteRate) - .hasSize(2) - .containsEntry(1, new BigDecimal("1.0")) - .containsEntry(2, new BigDecimal("10.0")); - - assertThat(wellKnownMetrics.brokerBytesOutFifteenMinuteRate) - .hasSize(2) - .containsEntry(1, new BigDecimal("2.0")) - .containsEntry(2, new BigDecimal("20.0")); - } - - @Test - void appliesInnerStateToMetricsBuilder() { - //filling per topic io rates - wellKnownMetrics.bytesInFifteenMinuteRate.put("topic", new BigDecimal(1)); - wellKnownMetrics.bytesOutFifteenMinuteRate.put("topic", new BigDecimal(2)); - - //filling per broker io rates - wellKnownMetrics.brokerBytesInFifteenMinuteRate.put(1, new BigDecimal(1)); - wellKnownMetrics.brokerBytesOutFifteenMinuteRate.put(1, new BigDecimal(2)); - wellKnownMetrics.brokerBytesInFifteenMinuteRate.put(2, new BigDecimal(10)); - wellKnownMetrics.brokerBytesOutFifteenMinuteRate.put(2, new BigDecimal(20)); - - Metrics.MetricsBuilder builder = Metrics.builder(); - wellKnownMetrics.apply(builder); - var metrics = builder.build(); - - // checking per topic io rates - assertThat(metrics.getTopicBytesInPerSec()).containsExactlyEntriesOf(wellKnownMetrics.bytesInFifteenMinuteRate); - assertThat(metrics.getTopicBytesOutPerSec()).containsExactlyEntriesOf(wellKnownMetrics.bytesOutFifteenMinuteRate); - - // checking per broker io rates - assertThat(metrics.getBrokerBytesInPerSec()).containsExactlyInAnyOrderEntriesOf( - Map.of(1, new BigDecimal(1), 2, new BigDecimal(10))); - assertThat(metrics.getBrokerBytesOutPerSec()).containsExactlyInAnyOrderEntriesOf( - Map.of(1, new BigDecimal(2), 2, new BigDecimal(20))); - } - - private void populateWith(Node n, String... prometheusMetric) { - Arrays.stream(prometheusMetric) - .map(PrometheusEndpointMetricsParser::parse) - .filter(Optional::isPresent) - .map(Optional::get) - .forEach(m -> wellKnownMetrics.populate(n, m)); - } - -} \ No newline at end of file diff --git a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/metrics/prometheus/PrometheusExposeTest.java b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/metrics/prometheus/PrometheusExposeTest.java new file mode 100644 index 00000000000..d4403872fbc --- /dev/null +++ b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/metrics/prometheus/PrometheusExposeTest.java @@ -0,0 +1,53 @@ +package com.provectus.kafka.ui.service.metrics.prometheus; + +import static com.provectus.kafka.ui.service.metrics.prometheus.PrometheusExpose.prepareMetricsForGlobalExpose; +import static io.prometheus.client.Collector.Type.GAUGE; +import static org.assertj.core.api.Assertions.assertThat; + +import com.provectus.kafka.ui.model.Metrics; +import com.provectus.kafka.ui.service.metrics.scrape.inferred.InferredMetrics; +import io.prometheus.client.Collector.MetricFamilySamples; +import io.prometheus.client.Collector.MetricFamilySamples.Sample; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; + +class PrometheusExposeTest { + + @Test + void prepareMetricsForGlobalExposeAppendsClusterAndBrokerIdLabelsToMetrics() { + + var inferredMfs = new MetricFamilySamples("infer", GAUGE, "help", List.of( + new Sample("infer1", List.of("lbl1"), List.of("lblVal1"), 100))); + + var broker1Mfs = new MetricFamilySamples("brok", GAUGE, "help", List.of( + new Sample("brok", List.of("broklbl1"), List.of("broklblVal1"), 101))); + + var broker2Mfs = new MetricFamilySamples("brok", GAUGE, "help", List.of( + new Sample("brok", List.of("broklbl1"), List.of("broklblVal1"), 102))); + + List prepared = prepareMetricsForGlobalExpose( + "testCluster", + Metrics.builder() + .inferredMetrics(new InferredMetrics(List.of(inferredMfs))) + .perBrokerScrapedMetrics(Map.of(1, List.of(broker1Mfs), 2, List.of(broker2Mfs))) + .build() + ).toList(); + + assertThat(prepared) + .hasSize(3) + .contains(new MetricFamilySamples("infer", GAUGE, "help", List.of( + new Sample("infer1", List.of("cluster", "lbl1"), List.of("testCluster", "lblVal1"), 100)))) + .contains( + new MetricFamilySamples("brok", GAUGE, "help", List.of( + new Sample("brok", List.of("cluster", "broker_id", "broklbl1"), + List.of("testCluster", "1", "broklblVal1"), 101))) + ) + .contains( + new MetricFamilySamples("brok", GAUGE, "help", List.of( + new Sample("brok", List.of("cluster", "broker_id", "broklbl1"), + List.of("testCluster", "2", "broklblVal1"), 102))) + ); + } + +} diff --git a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/metrics/scrape/IoRatesMetricsScannerTest.java b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/metrics/scrape/IoRatesMetricsScannerTest.java new file mode 100644 index 00000000000..8309b4e2d4d --- /dev/null +++ b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/metrics/scrape/IoRatesMetricsScannerTest.java @@ -0,0 +1,75 @@ +package com.provectus.kafka.ui.service.metrics.scrape; + +import static io.prometheus.client.Collector.MetricFamilySamples; +import static java.util.Arrays.stream; +import static java.util.stream.Collectors.toMap; +import static org.assertj.core.api.Assertions.assertThat; + +import com.provectus.kafka.ui.service.metrics.scrape.prometheus.PrometheusEndpointParser; +import java.math.BigDecimal; +import java.util.List; +import java.util.Map; +import org.apache.kafka.common.Node; +import org.junit.jupiter.api.Test; + +class IoRatesMetricsScannerTest { + + private IoRatesMetricsScanner ioRatesMetricsScanner; + + @Test + void bytesIoTopicMetricsPopulated() { + populateWith( + nodeMetrics( + new Node(0, "host", 123), + "kafka_server_BrokerTopicMetrics_FifteenMinuteRate{name=\"BytesInPerSec\",topic=\"test\",} 1.0", + "kafka_server_BrokerTopicMetrics_FifteenMinuteRate{name=\"BytesOutPerSec\",topic=\"test\",} 2.0", + "kafka_server_brokertopicmetrics_fifteenminuterate{name=\"bytesinpersec\",topic=\"test\",} 1.0", + "kafka_server_brokertopicmetrics_fifteenminuterate{name=\"bytesoutpersec\",topic=\"test\",} 2.0", + "some_unknown_prefix_brokertopicmetrics_fifteenminuterate{name=\"bytesinpersec\",topic=\"test\",} 1.0", + "some_unknown_prefix_brokertopicmetrics_fifteenminuterate{name=\"bytesoutpersec\",topic=\"test\",} 2.0" + ) + ); + assertThat(ioRatesMetricsScanner.bytesInFifteenMinuteRate) + .containsEntry("test", new BigDecimal("3.0")); + assertThat(ioRatesMetricsScanner.bytesOutFifteenMinuteRate) + .containsEntry("test", new BigDecimal("6.0")); + } + + @Test + void bytesIoBrokerMetricsPopulated() { + populateWith( + nodeMetrics( + new Node(1, "host1", 123), + "kafka_server_BrokerTopicMetrics_FifteenMinuteRate{name=\"BytesInPerSec\",} 1.0", + "kafka_server_BrokerTopicMetrics_FifteenMinuteRate{name=\"BytesOutPerSec\",} 2.0" + ), + nodeMetrics( + new Node(2, "host2", 345), + "some_unknown_prefix_brokertopicmetrics_fifteenminuterate{name=\"bytesinpersec\",} 10.0", + "some_unknown_prefix_brokertopicmetrics_fifteenminuterate{name=\"bytesoutpersec\",} 20.0" + ) + ); + + assertThat(ioRatesMetricsScanner.brokerBytesInFifteenMinuteRate) + .hasSize(2) + .containsEntry(1, new BigDecimal("1.0")) + .containsEntry(2, new BigDecimal("10.0")); + + assertThat(ioRatesMetricsScanner.brokerBytesOutFifteenMinuteRate) + .hasSize(2) + .containsEntry(1, new BigDecimal("2.0")) + .containsEntry(2, new BigDecimal("20.0")); + } + + @SafeVarargs + private void populateWith(Map.Entry>... entries) { + ioRatesMetricsScanner = new IoRatesMetricsScanner( + stream(entries).collect(toMap(Map.Entry::getKey, Map.Entry::getValue)) + ); + } + + private Map.Entry> nodeMetrics(Node n, String... prometheusMetrics) { + return Map.entry(n.id(), PrometheusEndpointParser.parse(stream(prometheusMetrics))); + } + +} diff --git a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/metrics/scrape/inferred/InferredMetricsScraperTest.java b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/metrics/scrape/inferred/InferredMetricsScraperTest.java new file mode 100644 index 00000000000..887a7555da4 --- /dev/null +++ b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/metrics/scrape/inferred/InferredMetricsScraperTest.java @@ -0,0 +1,121 @@ +package com.provectus.kafka.ui.service.metrics.scrape.inferred; + +import static com.provectus.kafka.ui.model.InternalLogDirStats.LogDirSpaceStats; +import static com.provectus.kafka.ui.model.InternalLogDirStats.SegmentStats; +import static com.provectus.kafka.ui.service.metrics.scrape.ScrapedClusterState.ConsumerGroupState; +import static com.provectus.kafka.ui.service.metrics.scrape.ScrapedClusterState.NodeState; +import static com.provectus.kafka.ui.service.metrics.scrape.ScrapedClusterState.TopicState; +import static org.assertj.core.api.Assertions.assertThat; + +import com.provectus.kafka.ui.model.InternalLogDirStats; +import com.provectus.kafka.ui.service.metrics.scrape.ScrapedClusterState; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import org.apache.kafka.clients.admin.ConsumerGroupDescription; +import org.apache.kafka.clients.admin.MemberAssignment; +import org.apache.kafka.clients.admin.MemberDescription; +import org.apache.kafka.clients.admin.TopicDescription; +import org.apache.kafka.common.Node; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.TopicPartitionInfo; +import org.junit.jupiter.api.Test; +import reactor.core.publisher.Mono; +import reactor.test.StepVerifier; + +class InferredMetricsScraperTest { + + final InferredMetricsScraper scraper = new InferredMetricsScraper(); + + @Test + void allExpectedMetricsScraped() { + var segmentStats = new SegmentStats(1234L, 3); + var logDirStats = new LogDirSpaceStats(234L, 345L, Map.of(), Map.of()); + + Node node1 = new Node(1, "node1", 9092); + Node node2 = new Node(2, "node2", 9092); + + Mono scraped = scraper.scrape( + ScrapedClusterState.builder() + .scrapeFinishedAt(Instant.now()) + .nodesStates( + Map.of( + 1, new NodeState(1, node1, segmentStats, logDirStats), + 2, new NodeState(2, node2, segmentStats, logDirStats) + ) + ) + .topicStates( + Map.of( + "t1", + new TopicState( + "t1", + new TopicDescription( + "t1", + false, + List.of( + new TopicPartitionInfo(0, node1, List.of(node1, node2), List.of(node1, node2)), + new TopicPartitionInfo(1, node1, List.of(node1, node2), List.of(node1)) + ) + ), + List.of(), + Map.of(0, 100L, 1, 101L), + Map.of(0, 200L, 1, 201L), + segmentStats, + Map.of(0, segmentStats, 1, segmentStats) + ) + ) + ) + .consumerGroupsStates( + Map.of( + "cg1", + new ConsumerGroupState( + "cg1", + new ConsumerGroupDescription( + "cg1", + true, + List.of( + new MemberDescription( + "memb1", Optional.empty(), "client1", "hst1", + new MemberAssignment(Set.of(new TopicPartition("t1", 0))) + ) + ), + null, + org.apache.kafka.common.ConsumerGroupState.STABLE, + node1 + ), + Map.of(new TopicPartition("t1", 0), 150L) + ) + ) + ) + .build() + ); + + StepVerifier.create(scraped) + .assertNext(inferredMetrics -> + assertThat(inferredMetrics.asStream().map(m -> m.name)).containsExactlyInAnyOrder( + "broker_count", + "broker_bytes_disk", + "broker_bytes_usable", + "broker_bytes_total", + "topic_count", + "kafka_topic_partitions", + "kafka_topic_partition_current_offset", + "kafka_topic_partition_oldest_offset", + "kafka_topic_partition_in_sync_replica", + "kafka_topic_partition_replicas", + "kafka_topic_partition_leader", + "topic_bytes_disk", + "group_count", + "group_state", + "group_member_count", + "group_host_count", + "kafka_consumergroup_current_offset", + "kafka_consumergroup_lag" + ) + ) + .verifyComplete(); + } + +} diff --git a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/metrics/JmxMetricsFormatterTest.java b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/metrics/scrape/jmx/JmxMetricsFormatterTest.java similarity index 95% rename from kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/metrics/JmxMetricsFormatterTest.java rename to kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/metrics/scrape/jmx/JmxMetricsFormatterTest.java index 1a4ff5134e6..fa4910f93d2 100644 --- a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/metrics/JmxMetricsFormatterTest.java +++ b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/metrics/scrape/jmx/JmxMetricsFormatterTest.java @@ -1,7 +1,8 @@ -package com.provectus.kafka.ui.service.metrics; +package com.provectus.kafka.ui.service.metrics.scrape.jmx; import static org.assertj.core.api.Assertions.assertThat; +import com.provectus.kafka.ui.service.metrics.RawMetric; import java.math.BigDecimal; import java.util.List; import java.util.Map; @@ -74,4 +75,4 @@ private void assertMetricsEqual(RawMetric expected, RawMetric actual) { assertThat(actual.value()).isCloseTo(expected.value(), Offset.offset(new BigDecimal("0.001"))); } -} \ No newline at end of file +} diff --git a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/metrics/scrape/prometheus/PrometheusEndpointParserTest.java b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/metrics/scrape/prometheus/PrometheusEndpointParserTest.java new file mode 100644 index 00000000000..6f7306723dc --- /dev/null +++ b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/metrics/scrape/prometheus/PrometheusEndpointParserTest.java @@ -0,0 +1,186 @@ +package com.provectus.kafka.ui.service.metrics.scrape.prometheus; + +import static com.provectus.kafka.ui.service.metrics.scrape.prometheus.PrometheusEndpointParser.parse; +import static io.prometheus.client.Collector.MetricFamilySamples; +import static io.prometheus.client.Collector.MetricFamilySamples.Sample; +import static io.prometheus.client.Collector.Type; +import static java.lang.Double.POSITIVE_INFINITY; +import static org.assertj.core.api.Assertions.assertThat; + +import com.google.common.collect.Iterators; +import com.google.common.collect.Lists; +import com.provectus.kafka.ui.service.metrics.prometheus.PrometheusExpose; +import io.prometheus.client.Collector; +import io.prometheus.client.CollectorRegistry; +import io.prometheus.client.Counter; +import io.prometheus.client.Gauge; +import io.prometheus.client.Histogram; +import io.prometheus.client.Info; +import io.prometheus.client.Summary; +import java.util.List; +import java.util.concurrent.ThreadLocalRandom; +import org.junit.jupiter.api.Test; + +class PrometheusEndpointParserTest { + + @Test + void parsesAllGeneratedMetricTypes() { + List original = generateMfs(); + String exposed = PrometheusExpose.constructHttpsResponse(original.stream()).getBody(); + List parsed = parse(exposed.lines()); + assertThat(parsed).containsExactlyElementsOf(original); + } + + @Test + void parsesMetricsFromPrometheusEndpointOutput() { + String expose = """ + # HELP http_requests_total The total number of HTTP requests. + # TYPE http_requests_total counter + http_requests_total{method="post",code="200",} 1027 1395066363000 + http_requests_total{method="post",code="400",} 3 1395066363000 + # Minimalistic line: + metric_without_timestamp_and_labels 12.47 + + # A weird metric from before the epoch: + something_weird{problem="division by zero"} +Inf -3982045 + + # TYPE something_untyped untyped + something_untyped{} -123123 + + # TYPE unit_test_seconds counter + # UNIT unit_test_seconds seconds + # HELP unit_test_seconds Testing that unit parsed properly + unit_test_seconds_total 4.20072246e+06 + + # HELP http_request_duration_seconds A histogram of the request duration. + # TYPE http_request_duration_seconds histogram + http_request_duration_seconds_bucket{le="0.05"} 24054 + http_request_duration_seconds_bucket{le="0.1"} 33444 + http_request_duration_seconds_bucket{le="0.2"} 100392 + http_request_duration_seconds_bucket{le="0.5"} 129389 + http_request_duration_seconds_bucket{le="1"} 133988 + http_request_duration_seconds_bucket{le="+Inf"} 144320 + http_request_duration_seconds_sum 53423 + http_request_duration_seconds_count 144320 + """; + List parsed = parse(expose.lines()); + assertThat(parsed).contains( + new MetricFamilySamples( + "http_requests_total", + Type.COUNTER, + "The total number of HTTP requests.", + List.of( + new Sample("http_requests_total", List.of("method", "code"), List.of("post", "200"), 1027), + new Sample("http_requests_total", List.of("method", "code"), List.of("post", "400"), 3) + ) + ), + new MetricFamilySamples( + "metric_without_timestamp_and_labels", + Type.GAUGE, + "metric_without_timestamp_and_labels", + List.of(new Sample("metric_without_timestamp_and_labels", List.of(), List.of(), 12.47)) + ), + new MetricFamilySamples( + "something_weird", + Type.GAUGE, + "something_weird", + List.of(new Sample("something_weird", List.of("problem"), List.of("division by zero"), POSITIVE_INFINITY)) + ), + new MetricFamilySamples( + "something_untyped", + Type.GAUGE, + "something_untyped", + List.of(new Sample("something_untyped", List.of(), List.of(), -123123)) + ), + new MetricFamilySamples( + "unit_test_seconds", + "seconds", + Type.COUNTER, + "Testing that unit parsed properly", + List.of(new Sample("unit_test_seconds_total", List.of(), List.of(), 4.20072246e+06)) + ), + new MetricFamilySamples( + "http_request_duration_seconds", + Type.HISTOGRAM, + "A histogram of the request duration.", + List.of( + new Sample("http_request_duration_seconds_bucket", List.of("le"), List.of("0.05"), 24054), + new Sample("http_request_duration_seconds_bucket", List.of("le"), List.of("0.1"), 33444), + new Sample("http_request_duration_seconds_bucket", List.of("le"), List.of("0.2"), 100392), + new Sample("http_request_duration_seconds_bucket", List.of("le"), List.of("0.5"), 129389), + new Sample("http_request_duration_seconds_bucket", List.of("le"), List.of("1"), 133988), + new Sample("http_request_duration_seconds_bucket", List.of("le"), List.of("+Inf"), 144320), + new Sample("http_request_duration_seconds_sum", List.of(), List.of(), 53423), + new Sample("http_request_duration_seconds_count", List.of(), List.of(), 144320) + ) + ) + ); + } + + private List generateMfs() { + CollectorRegistry collectorRegistry = new CollectorRegistry(); + + Gauge.build() + .name("test_gauge") + .help("help for gauge") + .register(collectorRegistry) + .set(42); + + Info.build() + .name("test_info") + .help("help for info") + .register(collectorRegistry) + .info("branch", "HEAD", "version", "1.2.3", "revision", "e0704b"); + + Counter.build() + .name("counter_no_labels") + .help("help for counter no lbls") + .register(collectorRegistry) + .inc(111); + + var counterWithLbls = Counter.build() + .name("counter_with_labels") + .help("help for counter with lbls") + .labelNames("lbl1", "lbl2") + .register(collectorRegistry); + + counterWithLbls.labels("v1", "v2").inc(234); + counterWithLbls.labels("v11", "v22").inc(345); + + var histogram = Histogram.build() + .name("test_hist") + .help("help for hist") + .linearBuckets(0.0, 1.0, 10) + .labelNames("lbl1", "lbl2") + .register(collectorRegistry); + + var summary = Summary.build() + .name("test_summary") + .help("help for hist") + .labelNames("lbl1", "lbl2") + .register(collectorRegistry); + + for (int i = 0; i < 30; i++) { + var val = ThreadLocalRandom.current().nextDouble(10.0); + histogram.labels("v1", "v2").observe(val); + summary.labels("v1", "v2").observe(val); + } + + //emulating unknown type + collectorRegistry.register(new Collector() { + @Override + public List collect() { + return List.of( + new MetricFamilySamples( + "test_unknown", + Type.UNKNOWN, + "help for unknown", + List.of(new Sample("test_unknown", List.of("l1"), List.of("v1"), 23432.0)) + ) + ); + } + }); + return Lists.newArrayList(Iterators.forEnumeration(collectorRegistry.metricFamilySamples())); + } + +} diff --git a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/metrics/scrape/prometheus/PrometheusMetricsRetrieverTest.java b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/metrics/scrape/prometheus/PrometheusMetricsRetrieverTest.java new file mode 100644 index 00000000000..7533385d203 --- /dev/null +++ b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/metrics/scrape/prometheus/PrometheusMetricsRetrieverTest.java @@ -0,0 +1,118 @@ +package com.provectus.kafka.ui.service.metrics.scrape.prometheus; + +import static io.prometheus.client.Collector.MetricFamilySamples; +import static io.prometheus.client.Collector.Type; +import static org.assertj.core.api.Assertions.assertThat; + +import com.provectus.kafka.ui.model.MetricsScrapeProperties; +import io.prometheus.client.Collector.MetricFamilySamples.Sample; +import java.io.IOException; +import java.util.List; +import okhttp3.mockwebserver.MockResponse; +import okhttp3.mockwebserver.MockWebServer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import reactor.test.StepVerifier; + +class PrometheusMetricsRetrieverTest { + + private final MockWebServer mockWebServer = new MockWebServer(); + + @BeforeEach + void startMockServer() throws IOException { + mockWebServer.start(); + } + + @AfterEach + void stopMockServer() throws IOException { + mockWebServer.close(); + } + + @Test + void callsMetricsEndpointAndConvertsResponceToRawMetric() { + var url = mockWebServer.url("/metrics"); + mockWebServer.enqueue(prepareResponse()); + + MetricsScrapeProperties scrapeProperties = prepareMetricsConfig(url.port(), null, null); + var retriever = new PrometheusMetricsRetriever(scrapeProperties); + + StepVerifier.create(retriever.retrieve(url.host())) + .assertNext(metrics -> assertThat(metrics).containsExactlyElementsOf(expectedMetrics())) + .verifyComplete(); + } + + @Test + void callsSecureMetricsEndpointAndConvertsResponceToRawMetric() { + var url = mockWebServer.url("/metrics"); + mockWebServer.enqueue(prepareResponse()); + + MetricsScrapeProperties scrapeProperties = prepareMetricsConfig(url.port(), "username", "password"); + var retriever = new PrometheusMetricsRetriever(scrapeProperties); + + StepVerifier.create(retriever.retrieve(url.host())) + .assertNext(metrics -> assertThat(metrics).containsExactlyElementsOf(expectedMetrics())) + .verifyComplete(); + } + + private MockResponse prepareResponse() { + // body copied from jmx exporter output + return new MockResponse().setBody( + """ + # HELP kafka_server_KafkaRequestHandlerPool_FifteenMinuteRate Attribute exposed for management + # TYPE kafka_server_KafkaRequestHandlerPool_FifteenMinuteRate untyped + kafka_server_KafkaRequestHandlerPool_FifteenMinuteRate{name="RequestHandlerAvgIdlePercent",} 0.898 + # HELP kafka_server_socket_server_metrics_request_size_avg The average size of requests sent. + # TYPE kafka_server_socket_server_metrics_request_size_avg untyped + kafka_server_socket_server_metrics_request_size_avg{listener="PLAIN",networkProcessor="1",} 101.1 + kafka_server_socket_server_metrics_request_size_avg{listener="PLAIN2",networkProcessor="5",} 202.2 + """ + ); + } + + private MetricsScrapeProperties prepareMetricsConfig(Integer port, String username, String password) { + return MetricsScrapeProperties.builder() + .ssl(false) + .port(port) + .username(username) + .password(password) + .build(); + } + + private List expectedMetrics() { + return List.of( + new MetricFamilySamples( + "kafka_server_KafkaRequestHandlerPool_FifteenMinuteRate", + Type.GAUGE, + "Attribute exposed for management", + List.of( + new Sample( + "kafka_server_KafkaRequestHandlerPool_FifteenMinuteRate", + List.of("name"), + List.of("RequestHandlerAvgIdlePercent"), + 0.898 + ) + ) + ), + new MetricFamilySamples( + "kafka_server_socket_server_metrics_request_size_avg", + Type.GAUGE, + "The average size of requests sent.", + List.of( + new Sample( + "kafka_server_socket_server_metrics_request_size_avg", + List.of("listener", "networkProcessor"), + List.of("PLAIN", "1"), + 101.1 + ), + new Sample( + "kafka_server_socket_server_metrics_request_size_avg", + List.of("listener", "networkProcessor"), + List.of("PLAIN2", "5"), + 202.2 + ) + ) + ) + ); + } +} diff --git a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/metrics/sink/PrometheusRemoteWriteSinkTest.java b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/metrics/sink/PrometheusRemoteWriteSinkTest.java new file mode 100644 index 00000000000..d643a6d18bb --- /dev/null +++ b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/metrics/sink/PrometheusRemoteWriteSinkTest.java @@ -0,0 +1,62 @@ +package com.provectus.kafka.ui.service.metrics.sink; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.provectus.kafka.ui.container.PrometheusContainer; +import io.prometheus.client.Collector; +import io.prometheus.client.Collector.MetricFamilySamples; +import io.prometheus.client.Collector.MetricFamilySamples.Sample; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import prometheus.query.ApiClient; +import prometheus.query.api.PrometheusClientApi; +import prometheus.query.model.QueryResponse; + +class PrometheusRemoteWriteSinkTest { + + private final PrometheusContainer prometheusContainer = new PrometheusContainer(); + + @BeforeEach + void startPromContainer() { + prometheusContainer.start(); + } + + @AfterEach + void stopPromContainer() { + prometheusContainer.stop(); + } + + @Test + void metricsPushedToPrometheus() { + var sink = new PrometheusRemoteWriteSink(prometheusContainer.url(), null); + sink.send( + Stream.of( + new MetricFamilySamples( + "test_metric1", Collector.Type.GAUGE, "help here", + List.of(new Sample("test_metric1", List.of(), List.of(), 111.111)) + ), + new MetricFamilySamples( + "test_metric2", Collector.Type.GAUGE, "help here", + List.of(new Sample("test_metric2", List.of(), List.of(), 222.222)) + ) + ) + ).block(); + + assertThat(queryMetricValue("test_metric1")) + .isEqualTo("111.111"); + + assertThat(queryMetricValue("test_metric2")) + .isEqualTo("222.222"); + } + + private String queryMetricValue(String metricName) { + PrometheusClientApi promClient = new PrometheusClientApi(new ApiClient().setBasePath(prometheusContainer.url())); + QueryResponse resp = promClient.query(metricName, null, null).block(); + return (String) ((List) ((Map) resp.getData().getResult().get(0)).get("value")).get(1); + } + +} diff --git a/kafka-ui-contract/pom.xml b/kafka-ui-contract/pom.xml index 0d8e238368f..e51612ec6f6 100644 --- a/kafka-ui-contract/pom.xml +++ b/kafka-ui-contract/pom.xml @@ -46,6 +46,11 @@ javax.annotation-api 1.3.2 + + com.google.protobuf + protobuf-java + 3.22.4 + @@ -151,6 +156,30 @@ + + generate-prometheus-query-api + + generate + + + ${project.basedir}/src/main/resources/swagger/prometheus-query-api.yaml + + ${project.build.directory}/generated-sources/prometheus-query-api + java + false + false + + prometheus.query.model + prometheus.query.api + prometheus-query + true + webclient + true + true + java8 + + + @@ -229,7 +258,40 @@ - + + + kr.motd.maven + os-maven-plugin + 1.7.0 + + + initialize + + detect + + + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + 0.6.1 + + + generate-sources + + compile + + + + + false + ${project.basedir}/src/main/resources/proto/prometheus-remote-write-api + + **/*.proto + + com.google.protobuf:protoc:3.21.12:exe:${os.detected.classifier} + diff --git a/kafka-ui-contract/src/main/resources/proto/prometheus-remote-write-api/gogoproto/gogo.proto b/kafka-ui-contract/src/main/resources/proto/prometheus-remote-write-api/gogoproto/gogo.proto new file mode 100644 index 00000000000..2f0a3c76bd5 --- /dev/null +++ b/kafka-ui-contract/src/main/resources/proto/prometheus-remote-write-api/gogoproto/gogo.proto @@ -0,0 +1,133 @@ +// Protocol Buffers for Go with Gadgets +// +// Copyright (c) 2013, The GoGo Authors. All rights reserved. +// http://github.com/gogo/protobuf +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto2"; +package gogoproto; + +import "google/protobuf/descriptor.proto"; + +option java_package = "com.google.protobuf"; +option java_outer_classname = "GoGoProtos"; +option go_package = "github.com/gogo/protobuf/gogoproto"; + +extend google.protobuf.EnumOptions { + optional bool goproto_enum_prefix = 62001; + optional bool goproto_enum_stringer = 62021; + optional bool enum_stringer = 62022; + optional string enum_customname = 62023; + optional bool enumdecl = 62024; +} + +extend google.protobuf.EnumValueOptions { + optional string enumvalue_customname = 66001; +} + +extend google.protobuf.FileOptions { + optional bool goproto_getters_all = 63001; + optional bool goproto_enum_prefix_all = 63002; + optional bool goproto_stringer_all = 63003; + optional bool verbose_equal_all = 63004; + optional bool face_all = 63005; + optional bool gostring_all = 63006; + optional bool populate_all = 63007; + optional bool stringer_all = 63008; + optional bool onlyone_all = 63009; + + optional bool equal_all = 63013; + optional bool description_all = 63014; + optional bool testgen_all = 63015; + optional bool benchgen_all = 63016; + optional bool marshaler_all = 63017; + optional bool unmarshaler_all = 63018; + optional bool stable_marshaler_all = 63019; + + optional bool sizer_all = 63020; + + optional bool goproto_enum_stringer_all = 63021; + optional bool enum_stringer_all = 63022; + + optional bool unsafe_marshaler_all = 63023; + optional bool unsafe_unmarshaler_all = 63024; + + optional bool goproto_extensions_map_all = 63025; + optional bool goproto_unrecognized_all = 63026; + optional bool gogoproto_import = 63027; + optional bool protosizer_all = 63028; + optional bool compare_all = 63029; + optional bool typedecl_all = 63030; + optional bool enumdecl_all = 63031; + + optional bool goproto_registration = 63032; +} + +extend google.protobuf.MessageOptions { + optional bool goproto_getters = 64001; + optional bool goproto_stringer = 64003; + optional bool verbose_equal = 64004; + optional bool face = 64005; + optional bool gostring = 64006; + optional bool populate = 64007; + optional bool stringer = 67008; + optional bool onlyone = 64009; + + optional bool equal = 64013; + optional bool description = 64014; + optional bool testgen = 64015; + optional bool benchgen = 64016; + optional bool marshaler = 64017; + optional bool unmarshaler = 64018; + optional bool stable_marshaler = 64019; + + optional bool sizer = 64020; + + optional bool unsafe_marshaler = 64023; + optional bool unsafe_unmarshaler = 64024; + + optional bool goproto_extensions_map = 64025; + optional bool goproto_unrecognized = 64026; + + optional bool protosizer = 64028; + optional bool compare = 64029; + + optional bool typedecl = 64030; +} + +extend google.protobuf.FieldOptions { + optional bool nullable = 65001; + optional bool embed = 65002; + optional string customtype = 65003; + optional string customname = 65004; + optional string jsontag = 65005; + optional string moretags = 65006; + optional string casttype = 65007; + optional string castkey = 65008; + optional string castvalue = 65009; + + optional bool stdtime = 65010; + optional bool stdduration = 65011; +} diff --git a/kafka-ui-contract/src/main/resources/proto/prometheus-remote-write-api/remote.proto b/kafka-ui-contract/src/main/resources/proto/prometheus-remote-write-api/remote.proto new file mode 100644 index 00000000000..0c50ce8234a --- /dev/null +++ b/kafka-ui-contract/src/main/resources/proto/prometheus-remote-write-api/remote.proto @@ -0,0 +1,88 @@ +// Copyright 2016 Prometheus Team +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; +package prometheus; + +option go_package = "prompb"; + +import "types.proto"; +import "gogoproto/gogo.proto"; + +message WriteRequest { + repeated prometheus.TimeSeries timeseries = 1 [(gogoproto.nullable) = false]; + // Cortex uses this field to determine the source of the write request. + // We reserve it to avoid any compatibility issues. + reserved 2; + repeated prometheus.MetricMetadata metadata = 3 [(gogoproto.nullable) = false]; +} + +// ReadRequest represents a remote read request. +message ReadRequest { + repeated Query queries = 1; + + enum ResponseType { + // Server will return a single ReadResponse message with matched series that includes list of raw samples. + // It's recommended to use streamed response types instead. + // + // Response headers: + // Content-Type: "application/x-protobuf" + // Content-Encoding: "snappy" + SAMPLES = 0; + // Server will stream a delimited ChunkedReadResponse message that + // contains XOR or HISTOGRAM(!) encoded chunks for a single series. + // Each message is following varint size and fixed size bigendian + // uint32 for CRC32 Castagnoli checksum. + // + // Response headers: + // Content-Type: "application/x-streamed-protobuf; proto=prometheus.ChunkedReadResponse" + // Content-Encoding: "" + STREAMED_XOR_CHUNKS = 1; + } + + // accepted_response_types allows negotiating the content type of the response. + // + // Response types are taken from the list in the FIFO order. If no response type in `accepted_response_types` is + // implemented by server, error is returned. + // For request that do not contain `accepted_response_types` field the SAMPLES response type will be used. + repeated ResponseType accepted_response_types = 2; +} + +// ReadResponse is a response when response_type equals SAMPLES. +message ReadResponse { + // In same order as the request's queries. + repeated QueryResult results = 1; +} + +message Query { + int64 start_timestamp_ms = 1; + int64 end_timestamp_ms = 2; + repeated prometheus.LabelMatcher matchers = 3; + prometheus.ReadHints hints = 4; +} + +message QueryResult { + // Samples within a time series must be ordered by time. + repeated prometheus.TimeSeries timeseries = 1; +} + +// ChunkedReadResponse is a response when response_type equals STREAMED_XOR_CHUNKS. +// We strictly stream full series after series, optionally split by time. This means that a single frame can contain +// partition of the single series, but once a new series is started to be streamed it means that no more chunks will +// be sent for previous one. Series are returned sorted in the same way TSDB block are internally. +message ChunkedReadResponse { + repeated prometheus.ChunkedSeries chunked_series = 1; + + // query_index represents an index of the query from ReadRequest.queries these chunks relates to. + int64 query_index = 2; +} diff --git a/kafka-ui-contract/src/main/resources/proto/prometheus-remote-write-api/types.proto b/kafka-ui-contract/src/main/resources/proto/prometheus-remote-write-api/types.proto new file mode 100644 index 00000000000..69bec49549d --- /dev/null +++ b/kafka-ui-contract/src/main/resources/proto/prometheus-remote-write-api/types.proto @@ -0,0 +1,187 @@ +// Copyright 2017 Prometheus Team +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; +package prometheus; + +option go_package = "prompb"; + +import "gogoproto/gogo.proto"; + +message MetricMetadata { + enum MetricType { + UNKNOWN = 0; + COUNTER = 1; + GAUGE = 2; + HISTOGRAM = 3; + GAUGEHISTOGRAM = 4; + SUMMARY = 5; + INFO = 6; + STATESET = 7; + } + + // Represents the metric type, these match the set from Prometheus. + // Refer to model/textparse/interface.go for details. + MetricType type = 1; + string metric_family_name = 2; + string help = 4; + string unit = 5; +} + +message Sample { + double value = 1; + // timestamp is in ms format, see model/timestamp/timestamp.go for + // conversion from time.Time to Prometheus timestamp. + int64 timestamp = 2; +} + +message Exemplar { + // Optional, can be empty. + repeated Label labels = 1 [(gogoproto.nullable) = false]; + double value = 2; + // timestamp is in ms format, see model/timestamp/timestamp.go for + // conversion from time.Time to Prometheus timestamp. + int64 timestamp = 3; +} + +// A native histogram, also known as a sparse histogram. +// Original design doc: +// https://docs.google.com/document/d/1cLNv3aufPZb3fNfaJgdaRBZsInZKKIHo9E6HinJVbpM/edit +// The appendix of this design doc also explains the concept of float +// histograms. This Histogram message can represent both, the usual +// integer histogram as well as a float histogram. +message Histogram { + enum ResetHint { + UNKNOWN = 0; // Need to test for a counter reset explicitly. + YES = 1; // This is the 1st histogram after a counter reset. + NO = 2; // There was no counter reset between this and the previous Histogram. + GAUGE = 3; // This is a gauge histogram where counter resets don't happen. + } + + oneof count { // Count of observations in the histogram. + uint64 count_int = 1; + double count_float = 2; + } + double sum = 3; // Sum of observations in the histogram. + // The schema defines the bucket schema. Currently, valid numbers + // are -4 <= n <= 8. They are all for base-2 bucket schemas, where 1 + // is a bucket boundary in each case, and then each power of two is + // divided into 2^n logarithmic buckets. Or in other words, each + // bucket boundary is the previous boundary times 2^(2^-n). In the + // future, more bucket schemas may be added using numbers < -4 or > + // 8. + sint32 schema = 4; + double zero_threshold = 5; // Breadth of the zero bucket. + oneof zero_count { // Count in zero bucket. + uint64 zero_count_int = 6; + double zero_count_float = 7; + } + + // Negative Buckets. + repeated BucketSpan negative_spans = 8 [(gogoproto.nullable) = false]; + // Use either "negative_deltas" or "negative_counts", the former for + // regular histograms with integer counts, the latter for float + // histograms. + repeated sint64 negative_deltas = 9; // Count delta of each bucket compared to previous one (or to zero for 1st bucket). + repeated double negative_counts = 10; // Absolute count of each bucket. + + // Positive Buckets. + repeated BucketSpan positive_spans = 11 [(gogoproto.nullable) = false]; + // Use either "positive_deltas" or "positive_counts", the former for + // regular histograms with integer counts, the latter for float + // histograms. + repeated sint64 positive_deltas = 12; // Count delta of each bucket compared to previous one (or to zero for 1st bucket). + repeated double positive_counts = 13; // Absolute count of each bucket. + + ResetHint reset_hint = 14; + // timestamp is in ms format, see model/timestamp/timestamp.go for + // conversion from time.Time to Prometheus timestamp. + int64 timestamp = 15; +} + +// A BucketSpan defines a number of consecutive buckets with their +// offset. Logically, it would be more straightforward to include the +// bucket counts in the Span. However, the protobuf representation is +// more compact in the way the data is structured here (with all the +// buckets in a single array separate from the Spans). +message BucketSpan { + sint32 offset = 1; // Gap to previous span, or starting point for 1st span (which can be negative). + uint32 length = 2; // Length of consecutive buckets. +} + +// TimeSeries represents samples and labels for a single time series. +message TimeSeries { + // For a timeseries to be valid, and for the samples and exemplars + // to be ingested by the remote system properly, the labels field is required. + repeated Label labels = 1 [(gogoproto.nullable) = false]; + repeated Sample samples = 2 [(gogoproto.nullable) = false]; + repeated Exemplar exemplars = 3 [(gogoproto.nullable) = false]; + repeated Histogram histograms = 4 [(gogoproto.nullable) = false]; +} + +message Label { + string name = 1; + string value = 2; +} + +message Labels { + repeated Label labels = 1 [(gogoproto.nullable) = false]; +} + +// Matcher specifies a rule, which can match or set of labels or not. +message LabelMatcher { + enum Type { + EQ = 0; + NEQ = 1; + RE = 2; + NRE = 3; + } + Type type = 1; + string name = 2; + string value = 3; +} + +message ReadHints { + int64 step_ms = 1; // Query step size in milliseconds. + string func = 2; // String representation of surrounding function or aggregation. + int64 start_ms = 3; // Start time in milliseconds. + int64 end_ms = 4; // End time in milliseconds. + repeated string grouping = 5; // List of label names used in aggregation. + bool by = 6; // Indicate whether it is without or by. + int64 range_ms = 7; // Range vector selector range in milliseconds. +} + +// Chunk represents a TSDB chunk. +// Time range [min, max] is inclusive. +message Chunk { + int64 min_time_ms = 1; + int64 max_time_ms = 2; + + // We require this to match chunkenc.Encoding. + enum Encoding { + UNKNOWN = 0; + XOR = 1; + HISTOGRAM = 2; + FLOAT_HISTOGRAM = 3; + } + Encoding type = 3; + bytes data = 4; +} + +// ChunkedSeries represents single, encoded time series. +message ChunkedSeries { + // Labels should be sorted. + repeated Label labels = 1 [(gogoproto.nullable) = false]; + // Chunks will be in start time order and may overlap. + repeated Chunk chunks = 2 [(gogoproto.nullable) = false]; +} diff --git a/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml b/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml index 306c3fd2ddd..c630ede81d5 100644 --- a/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml +++ b/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml @@ -1,4 +1,4 @@ -openapi: 3.0.0 +openapi: 3.0.1 info: description: Api Documentation version: 0.1.0 @@ -32,6 +32,52 @@ paths: $ref: '#/components/schemas/Cluster' + /api/clusters/{clusterName}/graphs/descriptions: + get: + tags: + - Graphs + summary: getGraphsList + operationId: getGraphsList + parameters: + - name: clusterName + in: path + required: true + schema: + type: string + responses: + 200: + description: | + Success + content: + application/json: + schema: + $ref: '#/components/schemas/GraphDescriptions' + + /api/clusters/{clusterName}/graphs/prometheus: + post: + tags: + - Graphs + summary: getGraphData + operationId: getGraphData + parameters: + - name: clusterName + in: path + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/GraphDataRequest' + responses: + 200: + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/PrometheusApiQueryResponse' + /api/clusters/{clusterName}/cache: post: tags: @@ -157,6 +203,20 @@ paths: schema: $ref: '#/components/schemas/ClusterMetrics' + /metrics: + get: + tags: + - PrometheusExpose + summary: getAllMetrics + operationId: getAllMetrics + responses: + 200: + description: OK + content: + application/text: + schema: + type: string + /api/clusters/{clusterName}/stats: get: tags: @@ -3740,6 +3800,112 @@ components: additionalProperties: $ref: '#/components/schemas/ClusterConfigValidation' + GraphDataRequest: + type: object + properties: + id: + type: string + parameters: + type: object + additionalProperties: + type: string + from: + type: string + format: date-time + to: + type: string + format: date-time + + GraphDescriptions: + type: object + properties: + graphs: + type: array + items: + $ref: '#/components/schemas/GraphDescription' + + GraphDescription: + type: object + required: ["id"] + properties: + id: + type: string + description: Id that should be used to query data on API level + type: + type: string + enum: ["range", "instant"] + defaultPeriod: + type: string + description: ISO_8601 duration string (for "range" graphs only) + parameters: + type: array + items: + $ref: '#/components/schemas/GraphParameter' + + GraphParameter: + type: object + required: ["name"] + properties: + name: + type: string + + PrometheusApiBaseResponse: + type: object + required: [ status ] + properties: + status: + type: string + enum: [ "success", "error" ] + error: + type: string + errorType: + type: string + warnings: + type: array + items: + type: string + + PrometheusApiQueryResponse: + type: object + allOf: + - $ref: "#/components/schemas/PrometheusApiBaseResponse" + properties: + data: + $ref: '#/components/schemas/PrometheusApiQueryResponseData' + + PrometheusApiQueryResponseData: + type: object + required: [ "resultType" ] + properties: + resultType: + type: string + enum: [ "matrix", "vector", "scalar", "string"] + result: + type: array + items: { } + description: | + Depending on resultType format can vary: + "vector": + [ + { + "metric": { "": "", ... }, + "value": [ , "" ], + "histogram": [ , ] + }, ... + ] + "matrix": + [ + { + "metric": { "": "", ... }, + "values": [ [ , "" ], ... ], + "histograms": [ [ , ], ... ] + }, ... + ] + "scalar": + [ , "" ] + "string": + [ , "" ] + ApplicationPropertyValidation: type: object required: [error] @@ -3764,6 +3930,8 @@ components: $ref: '#/components/schemas/ApplicationPropertyValidation' ksqldb: $ref: '#/components/schemas/ApplicationPropertyValidation' + prometheusStorage: + $ref: '#/components/schemas/ApplicationPropertyValidation' ApplicationConfig: type: object @@ -3960,6 +4128,31 @@ components: type: string keystorePassword: type: string + prometheusExpose: + type: boolean + store: + type: object + properties: + prometheus: + type: object + properties: + url: + type: string + remoteWrite: + type: boolean + pushGatewayUrl: + type: string + pushGatewayUsername: + type: string + pushGatewayPassword: + type: string + pushGatewayJobName: + type: string + kafka: + type: object + properties: + topic: + type: string properties: type: object additionalProperties: true diff --git a/kafka-ui-contract/src/main/resources/swagger/prometheus-query-api.yaml b/kafka-ui-contract/src/main/resources/swagger/prometheus-query-api.yaml new file mode 100644 index 00000000000..5804bf16088 --- /dev/null +++ b/kafka-ui-contract/src/main/resources/swagger/prometheus-query-api.yaml @@ -0,0 +1,365 @@ +openapi: 3.0.1 +info: + title: | + Prometheus query HTTP API + version: 0.1.0 + contact: { } + +tags: + - name: /promclient +servers: + - url: /localhost + + +paths: + /api/v1/label/{label_name}/values: + get: + tags: + - PrometheusClient + summary: Returns label values + description: "returns a list of label values for a provided label name" + operationId: getLabelValues + parameters: + - name: label_name + in: path + required: true + schema: + type: string + - name: start + in: query + description: Start timestamp. + schema: + type: string + format: rfc3339 | unix_timestamp + - name: end + in: query + description: End timestamp. + schema: + type: string + format: rfc3339 | unix_timestamp + - name: match[] + in: query + description: Repeated series selector argument that selects the series from which to read the label values. + schema: + type: string + format: series_selector + responses: + 200: + description: Success + content: + application/json: + schema: + $ref: '#/components/schemas/LabelValuesResponse' + + /api/v1/labels: + get: + tags: + - PrometheusClient + summary: Returns label names + description: returns a list of label names + operationId: getLabelNames + parameters: + - name: start + in: query + description: | + Start timestamp. + schema: + type: string + format: rfc3339 | unix_timestamp + - name: end + in: query + description: | + End timestamp. + schema: + type: string + format: rfc3339 | unix_timestamp + - name: match[] + in: query + description: Repeated series selector argument that selects the series from which to read the label values. Optional. + schema: + type: string + format: series_selector + responses: + 200: + description: Success + content: + application/json: + schema: + $ref: '#/components/schemas/LabelNamesResponse' + + /api/v1/metadata: + get: + tags: + - PrometheusClient + summary: Returns metric metadata + description: returns a list of label names + operationId: getMetricMetadata + parameters: + - name: limit + in: query + description: Maximum number of metrics to return. + required: true + schema: + type: integer + - name: metric + in: query + description: A metric name to filter metadata for. All metric metadata is retrieved if left empty. + schema: + type: string + responses: + 200: + description: Success + content: + application/json: + schema: + $ref: '#/components/schemas/MetadataResponse' + 201: + description: | + Success + content: + application/json: + schema: + $ref: '#/components/schemas/MetadataResponse' + + /api/v1/query: + get: + tags: + - PrometheusClient + summary: Evaluates instant query + description: | + Evaluates an instant query at a single point in time + operationId: query + parameters: + - name: query + in: query + description: | + Prometheus expression query string. + required: true + schema: + type: string + - name: time + in: query + description: | + Evaluation timestamp. Optional. + schema: + type: string + format: rfc3339 | unix_timestamp + - name: timeout + in: query + description: | + Evaluation timeout. Optional. + schema: + type: string + format: duration + responses: + 200: + description: | + Success + content: + application/json: + schema: + $ref: '#/components/schemas/QueryResponse' + + + /api/v1/query_range: + get: + tags: + - PrometheusClient + summary: Evaluates query over range of time. + description: Evaluates an expression query over a range of time + operationId: queryRange + parameters: + - name: query + in: query + description: Prometheus expression query string. + required: true + schema: + type: string + - name: start + in: query + description: Start timestamp. + schema: + type: string + format: rfc3339 | unix_timestamp + - name: end + in: query + description: End timestamp. + schema: + type: string + format: rfc3339 | unix_timestamp + - name: step + in: query + description: | + Query resolution step width in ```duration``` format or float number of seconds. + schema: + type: string + format: duration | float + - name: timeout + in: query + description: | + Evaluation timeout. Optional. Defaults to and is capped by the value of the ```-query.timeout``` flag. + schema: + type: string + format: duration + responses: + 200: + description: | + Success + content: + application/json: + schema: + $ref: "#/components/schemas/QueryResponse" + + + /api/v1/series: + get: + tags: + - PrometheusClient + summary: Returns time series + operationId: getSeries + parameters: + - name: start + in: query + description: | + Start timestamp. Optional. + schema: + type: string + format: rfc3339 | unix_timestamp + - name: end + in: query + description: | + End timestamp. Optional. + schema: + type: string + format: rfc3339 | unix_timestamp + - name: match[] + in: query + description: | + Repeated series selector argument that selects the series to return. At least one ```match[]``` argument must be provided. + required: true + schema: + type: string + format: series_selector + responses: + 200: + description: | + Success + content: + application/json: + schema: + $ref: '#/components/schemas/SeriesResponse' + +components: + schemas: + BaseResponse: + type: object + required: [ status ] + properties: + status: + type: string + enum: [ "success", "error" ] + error: + type: string + errorType: + type: string + warnings: + type: array + items: + type: string + + QueryResponse: + type: object + allOf: + - $ref: "#/components/schemas/BaseResponse" + properties: + data: + $ref: '#/components/schemas/QueryResponseData' + + QueryResponseData: + type: object + required: [ "resultType" ] + properties: + resultType: + type: string + enum: [ "matrix", "vector", "scalar", "string"] + result: + type: array + items: { } + description: | + Depending on resultType format can vary: + "vector": + [ + { + "metric": { "": "", ... }, + "value": [ , "" ], + "histogram": [ , ] + }, ... + ] + "matrix": + [ + { + "metric": { "": "", ... }, + "values": [ [ , "" ], ... ], + "histograms": [ [ , ], ... ] + }, ... + ] + "scalar": + [ , "" ] + "string": + [ , "" ] + + SeriesResponse: + type: object + allOf: + - $ref: "#/components/schemas/BaseResponse" + properties: + data: + type: array + description: a list of objects that contain the label name/value pairs which + identify each series + items: + type: object + properties: + __name__: + type: string + job: + type: string + instance: + type: string + + MetadataResponse: + type: object + allOf: + - $ref: "#/components/schemas/BaseResponse" + properties: + data: + type: object + additionalProperties: + type: array + items: + type: object + additionalProperties: true + + LabelValuesResponse: + type: object + allOf: + - $ref: "#/components/schemas/BaseResponse" + properties: + data: + type: array + description: a list of string label values + items: + type: string + + LabelNamesResponse: + type: object + allOf: + - $ref: "#/components/schemas/BaseResponse" + properties: + data: + type: array + description: a list of string label names + items: + type: string + +