From fd5fedfea45dd690820eba63f666b4dcd654adc3 Mon Sep 17 00:00:00 2001 From: NOZAIS Julien Date: Thu, 26 Sep 2024 16:09:03 +0200 Subject: [PATCH 01/34] allow development behind proxy --- .mvn/jvm.config | 1 + api/pom.xml | 1 + contract/pom.xml | 1 + pom.xml | 16 ++++++++-------- 4 files changed, 11 insertions(+), 8 deletions(-) create mode 100644 .mvn/jvm.config diff --git a/.mvn/jvm.config b/.mvn/jvm.config new file mode 100644 index 000000000..2bf66750a --- /dev/null +++ b/.mvn/jvm.config @@ -0,0 +1 @@ +-Djava.net.useSystemProxies=true \ No newline at end of file diff --git a/api/pom.xml b/api/pom.xml index a892eba7f..57e3c6a7e 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -502,6 +502,7 @@ build + false diff --git a/contract/pom.xml b/contract/pom.xml index 8d7e76cea..55d86d40b 100644 --- a/contract/pom.xml +++ b/contract/pom.xml @@ -201,6 +201,7 @@ gen:sources + false diff --git a/pom.xml b/pom.xml index 679587ce6..b384d9684 100644 --- a/pom.xml +++ b/pom.xml @@ -74,10 +74,6 @@ - - confluent - https://packages.confluent.io/maven/ - central Central Repository @@ -87,13 +83,13 @@ false + + confluent + https://packages.confluent.io/maven/ + - - confluent - https://packages.confluent.io/maven/ - central Central Repository @@ -106,6 +102,10 @@ never + + confluent + https://packages.confluent.io/maven/ + From 70517a0ed1872f58d5b582d9c39a4780e7969643 Mon Sep 17 00:00:00 2001 From: NOZAIS Julien Date: Mon, 30 Sep 2024 16:55:19 +0200 Subject: [PATCH 02/34] Add dropdown options to pause and stop connectors --- .../ui/client/RetryingKafkaConnectClient.java | 5 +++ .../ui/service/KafkaConnectService.java | 2 ++ .../main/resources/swagger/kafbat-ui-api.yaml | 14 ++++++-- .../resources/swagger/kafka-connect-api.yaml | 16 +++++++++ .../Connect/Details/Actions/Actions.tsx | 17 +++++++++- .../components/Connect/List/ActionsCell.tsx | 34 ++++++++++++++++++- 6 files changed, 84 insertions(+), 4 deletions(-) diff --git a/api/src/main/java/io/kafbat/ui/client/RetryingKafkaConnectClient.java b/api/src/main/java/io/kafbat/ui/client/RetryingKafkaConnectClient.java index bb4ea92fe..2a13cb961 100644 --- a/api/src/main/java/io/kafbat/ui/client/RetryingKafkaConnectClient.java +++ b/api/src/main/java/io/kafbat/ui/client/RetryingKafkaConnectClient.java @@ -190,6 +190,11 @@ public Mono pauseConnector(String connectorName) throws WebClientResponseE return withRetryOnConflict(super.pauseConnector(connectorName)); } + @Override + public Mono stopConnector(String connectorName) throws WebClientResponseException { + return withRetryOnConflict(super.stopConnector(connectorName)); + } + @Override public Mono> pauseConnectorWithHttpInfo(String connectorName) throws WebClientResponseException { return withRetryOnConflict(super.pauseConnectorWithHttpInfo(connectorName)); diff --git a/api/src/main/java/io/kafbat/ui/service/KafkaConnectService.java b/api/src/main/java/io/kafbat/ui/service/KafkaConnectService.java index c5a32b29e..f55efcddf 100644 --- a/api/src/main/java/io/kafbat/ui/service/KafkaConnectService.java +++ b/api/src/main/java/io/kafbat/ui/service/KafkaConnectService.java @@ -229,6 +229,8 @@ public Mono updateConnectorState(KafkaCluster cluster, String connectName, t -> t.getStatus().getState() == ConnectorTaskStatusDTO.FAILED); case PAUSE: return client.pauseConnector(connectorName); + case STOP: + return client.stopConnector(connectorName); case RESUME: return client.resumeConnector(connectorName); default: diff --git a/contract/src/main/resources/swagger/kafbat-ui-api.yaml b/contract/src/main/resources/swagger/kafbat-ui-api.yaml index 7ca62831f..f9608e05e 100644 --- a/contract/src/main/resources/swagger/kafbat-ui-api.yaml +++ b/contract/src/main/resources/swagger/kafbat-ui-api.yaml @@ -1542,7 +1542,7 @@ paths: post: tags: - Kafka Connect - summary: update connector state (restart, pause or resume) + summary: update connector state (restart, pause, stop or resume) operationId: updateConnectorState parameters: - name: clusterName @@ -3482,6 +3482,7 @@ components: - RESTART_FAILED_TASKS - PAUSE - RESUME + - STOP TaskAction: type: string @@ -3867,7 +3868,16 @@ components: KafkaAcl: type: object - required: [resourceType, resourceName, namePatternType, principal, host, operation, permission] + required: + [ + resourceType, + resourceName, + namePatternType, + principal, + host, + operation, + permission, + ] properties: resourceType: $ref: '#/components/schemas/KafkaAclResourceType' diff --git a/contract/src/main/resources/swagger/kafka-connect-api.yaml b/contract/src/main/resources/swagger/kafka-connect-api.yaml index e014d5529..6335025e0 100644 --- a/contract/src/main/resources/swagger/kafka-connect-api.yaml +++ b/contract/src/main/resources/swagger/kafka-connect-api.yaml @@ -230,6 +230,22 @@ paths: 202: description: Accepted + /connectors/{connectorName}/stop: + put: + tags: + - KafkaConnectClient + summary: stop the connector + operationId: stopConnector + parameters: + - name: connectorName + in: path + required: true + schema: + type: string + responses: + 204: + description: No Content + /connectors/{connectorName}/tasks: get: tags: diff --git a/frontend/src/components/Connect/Details/Actions/Actions.tsx b/frontend/src/components/Connect/Details/Actions/Actions.tsx index 61eeabda4..1948dbf04 100644 --- a/frontend/src/components/Connect/Details/Actions/Actions.tsx +++ b/frontend/src/components/Connect/Details/Actions/Actions.tsx @@ -59,6 +59,8 @@ const Actions: React.FC = () => { stateMutation.mutateAsync(ConnectorAction.RESTART_FAILED_TASKS); const pauseConnectorHandler = () => stateMutation.mutateAsync(ConnectorAction.PAUSE); + const stopConnectorHandler = () => + stateMutation.mutateAsync(ConnectorAction.STOP); const resumeConnectorHandler = () => stateMutation.mutateAsync(ConnectorAction.RESUME); return ( @@ -84,7 +86,20 @@ const Actions: React.FC = () => { Pause )} - {connector?.status.state === ConnectorState.PAUSED && ( + {connector?.status.state === ConnectorState.RUNNING && ( + + Stop Connector + + )} + {(connector?.status.state === ConnectorState.PAUSED || connector?.status.state === ConnectorState.STOPPED) && ( > = ({ const restartFailedTasksHandler = () => stateMutation.mutateAsync(ConnectorAction.RESTART_FAILED_TASKS); + const pauseConnectorHandler = () => + stateMutation.mutateAsync(ConnectorAction.PAUSE); + + const stopConnectorHandler = () => + stateMutation.mutateAsync(ConnectorAction.STOP); + return ( - {status.state === ConnectorState.PAUSED && ( + {(status.state === ConnectorState.PAUSED || status.state === ConnectorState.STOPPED) && ( > = ({ Resume )} + {status.state === ConnectorState.RUNNING && ( + + Pause Connector + + )} + {status.state === ConnectorState.RUNNING && ( + + Stop Connector + + )} Date: Mon, 30 Sep 2024 16:58:02 +0200 Subject: [PATCH 03/34] format kafkbat-ui openapi definition --- .../main/resources/swagger/kafbat-ui-api.yaml | 95 ++++++++----------- 1 file changed, 37 insertions(+), 58 deletions(-) diff --git a/contract/src/main/resources/swagger/kafbat-ui-api.yaml b/contract/src/main/resources/swagger/kafbat-ui-api.yaml index f9608e05e..6e4d8bffc 100644 --- a/contract/src/main/resources/swagger/kafbat-ui-api.yaml +++ b/contract/src/main/resources/swagger/kafbat-ui-api.yaml @@ -4,7 +4,7 @@ info: version: 0.1.0 title: Api Documentation termsOfService: urn:tos - contact: { } + contact: {} license: name: Apache 2.0 url: https://www.apache.org/licenses/LICENSE-2.0 @@ -31,7 +31,6 @@ paths: items: $ref: '#/components/schemas/Cluster' - /api/clusters/{clusterName}/cache: post: tags: @@ -54,7 +53,6 @@ paths: 404: description: Not found - /api/clusters/{clusterName}/brokers: get: tags: @@ -432,7 +430,6 @@ paths: 404: description: Not found - /api/clusters/{clusterName}/topics/{topicName}: get: tags: @@ -644,7 +641,6 @@ paths: schema: $ref: '#/components/schemas/SmartFilterTestExecutionResult' - /api/clusters/{clusterName}/topics/{topicName}/messages: get: tags: @@ -665,7 +661,7 @@ paths: - name: seekType in: query schema: - $ref: "#/components/schemas/SeekType" + $ref: '#/components/schemas/SeekType' - name: seekTo in: query schema: @@ -684,19 +680,19 @@ paths: - name: filterQueryType in: query schema: - $ref: "#/components/schemas/MessageFilterType" + $ref: '#/components/schemas/MessageFilterType' - name: seekDirection in: query schema: - $ref: "#/components/schemas/SeekDirection" + $ref: '#/components/schemas/SeekDirection' - name: keySerde in: query - description: "Serde that should be used for deserialization. Will be chosen automatically if not set." + description: 'Serde that should be used for deserialization. Will be chosen automatically if not set.' schema: type: string - name: valueSerde in: query - description: "Serde that should be used for deserialization. Will be chosen automatically if not set." + description: 'Serde that should be used for deserialization. Will be chosen automatically if not set.' schema: type: string responses: @@ -793,7 +789,6 @@ paths: schema: $ref: '#/components/schemas/MessageFilterId' - /api/clusters/{clusterName}/topics/{topicName}/messages/v2: get: tags: @@ -815,7 +810,7 @@ paths: in: query description: Messages polling mode schema: - $ref: "#/components/schemas/PollingMode" + $ref: '#/components/schemas/PollingMode' - name: partitions in: query schema: @@ -852,17 +847,17 @@ paths: format: int64 - name: keySerde in: query - description: "Serde that should be used for deserialization. Will be chosen automatically if not set." + description: 'Serde that should be used for deserialization. Will be chosen automatically if not set.' schema: type: string - name: valueSerde in: query - description: "Serde that should be used for deserialization. Will be chosen automatically if not set." + description: 'Serde that should be used for deserialization. Will be chosen automatically if not set.' schema: type: string - name: cursor in: query - description: "id of the cursor for pagination, if passed - all other query params ignored" + description: 'id of the cursor for pagination, if passed - all other query params ignored' schema: type: string responses: @@ -875,7 +870,6 @@ paths: items: $ref: '#/components/schemas/TopicMessageEvent' - /api/clusters/{clusterName}/topics/{topicName}/activeproducers: get: tags: @@ -903,7 +897,6 @@ paths: items: $ref: '#/components/schemas/TopicProducerState' - /api/clusters/{clusterName}/topics/{topicName}/consumer-groups: get: tags: @@ -976,7 +969,6 @@ paths: schema: $ref: '#/components/schemas/ConsumerGroupsPageResponse' - /api/clusters/{clusterName}/consumer-groups/{id}: get: tags: @@ -1208,7 +1200,6 @@ paths: 404: description: Not found - /api/clusters/{clusterName}/schemas/{subject}/versions/{version}: get: tags: @@ -1699,7 +1690,6 @@ paths: 200: description: OK - /api/clusters/{clusterName}/ksql/v2: post: tags: @@ -2194,7 +2184,6 @@ paths: schema: $ref: '#/components/schemas/ApplicationConfigValidation' - /api/config/relatedfiles: post: tags: @@ -2240,7 +2229,7 @@ components: description: type: string preferred: - description: "This serde was automatically chosen by cluster config. This should be enabled in UI by default. Also it will be used for deserialization if no serdes passed." + description: 'This serde was automatically chosen by cluster config. This should be enabled in UI by default. Also it will be used for deserialization if no serdes passed.' type: boolean schema: type: string @@ -2527,7 +2516,7 @@ components: partitions: type: array items: - $ref: "#/components/schemas/Partition" + $ref: '#/components/schemas/Partition' required: - name @@ -2571,7 +2560,7 @@ components: partitionStats: type: array items: - $ref: "#/components/schemas/TopicAnalysisStats" + $ref: '#/components/schemas/TopicAnalysisStats' TopicAnalysisStats: type: object @@ -2579,7 +2568,7 @@ components: partition: type: integer format: int32 - description: "null if this is total stats" + description: 'null if this is total stats' totalMsgs: type: integer format: int64 @@ -2608,9 +2597,9 @@ components: type: integer format: int64 keySize: - $ref: "#/components/schemas/TopicAnalysisSizeStats" + $ref: '#/components/schemas/TopicAnalysisSizeStats' valueSize: - $ref: "#/components/schemas/TopicAnalysisSizeStats" + $ref: '#/components/schemas/TopicAnalysisSizeStats' hourlyMsgCounts: type: array items: @@ -2625,7 +2614,7 @@ components: TopicAnalysisSizeStats: type: object - description: "All sizes in bytes" + description: 'All sizes in bytes' properties: sum: type: integer @@ -2675,7 +2664,7 @@ components: partitions: type: array items: - $ref: "#/components/schemas/Partition" + $ref: '#/components/schemas/Partition' partitionCount: type: integer replicationFactor: @@ -2714,7 +2703,7 @@ components: defaultValue: type: string source: - $ref: "#/components/schemas/ConfigSource" + $ref: '#/components/schemas/ConfigSource' isSensitive: type: boolean isReadOnly: @@ -2722,7 +2711,7 @@ components: synonyms: type: array items: - $ref: "#/components/schemas/ConfigSynonym" + $ref: '#/components/schemas/ConfigSynonym' doc: type: string required: @@ -2838,7 +2827,7 @@ components: discriminator: propertyName: inherit mapping: - details: "#/components/schemas/ConsumerGroupDetails" + details: '#/components/schemas/ConsumerGroupDetails' type: object properties: groupId: @@ -2852,9 +2841,9 @@ components: partitionAssignor: type: string state: - $ref: "#/components/schemas/ConsumerGroupState" + $ref: '#/components/schemas/ConsumerGroupState' coordinator: - $ref: "#/components/schemas/Broker" + $ref: '#/components/schemas/Broker' consumerLag: type: integer format: int64 @@ -2949,13 +2938,13 @@ components: # otherwise ui should pass cursor param to continue polling - DONE message: - $ref: "#/components/schemas/TopicMessage" + $ref: '#/components/schemas/TopicMessage' phase: - $ref: "#/components/schemas/TopicMessagePhase" + $ref: '#/components/schemas/TopicMessagePhase' consuming: - $ref: "#/components/schemas/TopicMessageConsuming" + $ref: '#/components/schemas/TopicMessageConsuming' cursor: - $ref: "#/components/schemas/TopicMessageNextPageCursor" + $ref: '#/components/schemas/TopicMessageNextPageCursor' TopicMessagePhase: type: object @@ -3018,10 +3007,10 @@ components: type: string keyFormat: #deprecated - wont be filled - use 'keySerde' field instead - $ref: "#/components/schemas/MessageFormat" + $ref: '#/components/schemas/MessageFormat' valueFormat: #deprecated - wont be filled - use 'valueSerde' field instead - $ref: "#/components/schemas/MessageFormat" + $ref: '#/components/schemas/MessageFormat' keySize: type: integer format: int64 @@ -3147,7 +3136,6 @@ components: - topic - partition - ConsumerGroupDetails: allOf: - $ref: '#/components/schemas/ConsumerGroup' @@ -3868,16 +3856,7 @@ components: KafkaAcl: type: object - required: - [ - resourceType, - resourceName, - namePatternType, - principal, - host, - operation, - permission, - ] + required: [resourceType, resourceName, namePatternType, principal, host, operation, permission] properties: resourceType: $ref: '#/components/schemas/KafkaAclResourceType' @@ -3894,15 +3873,15 @@ components: enum: - UNKNOWN # Unknown operation, need to update mapping code on BE - ALL # Cluster, Topic, Group - - READ # Topic, Group + - READ # Topic, Group - WRITE # Topic, TransactionalId - CREATE # Cluster, Topic - - DELETE # Topic, Group - - ALTER # Cluster, Topic, + - DELETE # Topic, Group + - ALTER # Cluster, Topic, - DESCRIBE # Cluster, Topic, Group, TransactionalId, DelegationToken - CLUSTER_ACTION # Cluster - DESCRIBE_CONFIGS # Cluster, Topic - - ALTER_CONFIGS # Cluster, Topic + - ALTER_CONFIGS # Cluster, Topic - IDEMPOTENT_WRITE # Cluster - CREATE_TOKENS - DESCRIBE_TOKENS @@ -4133,7 +4112,7 @@ components: properties: maxInMemoryBufferSize: type: string - description: "examples: 20, 12KB, 5MB" + description: 'examples: 20, 12KB, 5MB' kafka: type: object properties: @@ -4190,7 +4169,7 @@ components: keystoreLocation: type: string keystorePassword: - type: string + type: string ksqldbServerAuth: type: object properties: @@ -4298,7 +4277,7 @@ components: properties: level: type: string - enum: [ "ALL", "ALTER_ONLY" ] + enum: ['ALL', 'ALTER_ONLY'] topic: type: string auditTopicsPartitions: From 503a2a1efb13e2806f8392abdaa048c2813dee78 Mon Sep 17 00:00:00 2001 From: NOZAIS Julien Date: Mon, 30 Sep 2024 18:58:49 +0200 Subject: [PATCH 04/34] Add a dropdown option to reset the offsets of stopped connectors --- .../ui/controller/KafkaConnectController.java | 119 +++++++----- .../model/rbac/permission/ConnectAction.java | 5 +- .../ui/service/KafkaConnectService.java | 174 +++++++++--------- .../main/resources/swagger/kafbat-ui-api.yaml | 26 +++ .../resources/swagger/kafka-connect-api.yaml | 31 +++- .../Connect/Details/Actions/Actions.tsx | 26 +++ .../components/Connect/List/ActionsCell.tsx | 31 +++- frontend/src/lib/hooks/api/kafkaConnect.ts | 8 + 8 files changed, 274 insertions(+), 146 deletions(-) diff --git a/api/src/main/java/io/kafbat/ui/controller/KafkaConnectController.java b/api/src/main/java/io/kafbat/ui/controller/KafkaConnectController.java index 08eb304c0..fd6838ab7 100644 --- a/api/src/main/java/io/kafbat/ui/controller/KafkaConnectController.java +++ b/api/src/main/java/io/kafbat/ui/controller/KafkaConnectController.java @@ -34,15 +34,15 @@ @RequiredArgsConstructor @Slf4j public class KafkaConnectController extends AbstractController implements KafkaConnectApi { - private static final Set RESTART_ACTIONS - = Set.of(RESTART, RESTART_FAILED_TASKS, RESTART_ALL_TASKS); + private static final Set RESTART_ACTIONS = Set.of(RESTART, RESTART_FAILED_TASKS, + RESTART_ALL_TASKS); private static final String CONNECTOR_NAME = "connectorName"; private final KafkaConnectService kafkaConnectService; @Override public Mono>> getConnects(String clusterName, - ServerWebExchange exchange) { + ServerWebExchange exchange) { Flux availableConnects = kafkaConnectService.getConnects(getCluster(clusterName)) .filterWhen(dto -> accessControlService.isConnectAccessible(dto, clusterName)); @@ -52,7 +52,7 @@ public Mono>> getConnects(String clusterName, @Override public Mono>> getConnectors(String clusterName, String connectName, - ServerWebExchange exchange) { + ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) @@ -61,14 +61,15 @@ public Mono>> getConnectors(String clusterName, Stri .build(); return validateAccess(context) - .thenReturn(ResponseEntity.ok(kafkaConnectService.getConnectorNames(getCluster(clusterName), connectName))) + .thenReturn( + ResponseEntity.ok(kafkaConnectService.getConnectorNames(getCluster(clusterName), connectName))) .doOnEach(sig -> audit(context, sig)); } @Override public Mono> createConnector(String clusterName, String connectName, - @Valid Mono connector, - ServerWebExchange exchange) { + @Valid Mono connector, + ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) @@ -78,14 +79,14 @@ public Mono> createConnector(String clusterName, St return validateAccess(context).then( kafkaConnectService.createConnector(getCluster(clusterName), connectName, connector) - .map(ResponseEntity::ok) - ).doOnEach(sig -> audit(context, sig)); + .map(ResponseEntity::ok)) + .doOnEach(sig -> audit(context, sig)); } @Override public Mono> getConnector(String clusterName, String connectName, - String connectorName, - ServerWebExchange exchange) { + String connectorName, + ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) @@ -95,14 +96,14 @@ public Mono> getConnector(String clusterName, Strin return validateAccess(context).then( kafkaConnectService.getConnector(getCluster(clusterName), connectName, connectorName) - .map(ResponseEntity::ok) - ).doOnEach(sig -> audit(context, sig)); + .map(ResponseEntity::ok)) + .doOnEach(sig -> audit(context, sig)); } @Override public Mono> deleteConnector(String clusterName, String connectName, - String connectorName, - ServerWebExchange exchange) { + String connectorName, + ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) @@ -113,19 +114,17 @@ public Mono> deleteConnector(String clusterName, String con return validateAccess(context).then( kafkaConnectService.deleteConnector(getCluster(clusterName), connectName, connectorName) - .map(ResponseEntity::ok) - ).doOnEach(sig -> audit(context, sig)); + .map(ResponseEntity::ok)) + .doOnEach(sig -> audit(context, sig)); } - @Override public Mono>> getAllConnectors( String clusterName, String search, ConnectorColumnsToSortDTO orderBy, SortOrderDTO sortOrder, - ServerWebExchange exchange - ) { + ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) .operationName("getAllConnectors") @@ -145,9 +144,9 @@ public Mono>> getAllConnectors( @Override public Mono>> getConnectorConfig(String clusterName, - String connectName, - String connectorName, - ServerWebExchange exchange) { + String connectName, + String connectorName, + ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) @@ -158,15 +157,15 @@ public Mono>> getConnectorConfig(String clust return validateAccess(context).then( kafkaConnectService .getConnectorConfig(getCluster(clusterName), connectName, connectorName) - .map(ResponseEntity::ok) - ).doOnEach(sig -> audit(context, sig)); + .map(ResponseEntity::ok)) + .doOnEach(sig -> audit(context, sig)); } @Override public Mono> setConnectorConfig(String clusterName, String connectName, - String connectorName, - Mono> requestBody, - ServerWebExchange exchange) { + String connectorName, + Mono> requestBody, + ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) @@ -176,22 +175,22 @@ public Mono> setConnectorConfig(String clusterName, .build(); return validateAccess(context).then( - kafkaConnectService - .setConnectorConfig(getCluster(clusterName), connectName, connectorName, requestBody) - .map(ResponseEntity::ok)) + kafkaConnectService + .setConnectorConfig(getCluster(clusterName), connectName, connectorName, requestBody) + .map(ResponseEntity::ok)) .doOnEach(sig -> audit(context, sig)); } @Override public Mono> updateConnectorState(String clusterName, String connectName, - String connectorName, - ConnectorActionDTO action, - ServerWebExchange exchange) { + String connectorName, + ConnectorActionDTO action, + ServerWebExchange exchange) { ConnectAction[] connectActions; if (RESTART_ACTIONS.contains(action)) { - connectActions = new ConnectAction[] {ConnectAction.VIEW, ConnectAction.RESTART}; + connectActions = new ConnectAction[] { ConnectAction.VIEW, ConnectAction.RESTART }; } else { - connectActions = new ConnectAction[] {ConnectAction.VIEW, ConnectAction.EDIT}; + connectActions = new ConnectAction[] { ConnectAction.VIEW, ConnectAction.EDIT }; } var context = AccessContext.builder() @@ -204,15 +203,15 @@ public Mono> updateConnectorState(String clusterName, Strin return validateAccess(context).then( kafkaConnectService .updateConnectorState(getCluster(clusterName), connectName, connectorName, action) - .map(ResponseEntity::ok) - ).doOnEach(sig -> audit(context, sig)); + .map(ResponseEntity::ok)) + .doOnEach(sig -> audit(context, sig)); } @Override public Mono>> getConnectorTasks(String clusterName, - String connectName, - String connectorName, - ServerWebExchange exchange) { + String connectName, + String connectorName, + ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) .connectActions(connectName, ConnectAction.VIEW) @@ -223,14 +222,14 @@ public Mono>> getConnectorTasks(String clusterName, return validateAccess(context).thenReturn( ResponseEntity .ok(kafkaConnectService - .getConnectorTasks(getCluster(clusterName), connectName, connectorName)) - ).doOnEach(sig -> audit(context, sig)); + .getConnectorTasks(getCluster(clusterName), connectName, connectorName))) + .doOnEach(sig -> audit(context, sig)); } @Override public Mono> restartConnectorTask(String clusterName, String connectName, - String connectorName, Integer taskId, - ServerWebExchange exchange) { + String connectorName, Integer taskId, + ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) @@ -242,8 +241,8 @@ public Mono> restartConnectorTask(String clusterName, Strin return validateAccess(context).then( kafkaConnectService .restartConnectorTask(getCluster(clusterName), connectName, connectorName, taskId) - .map(ResponseEntity::ok) - ).doOnEach(sig -> audit(context, sig)); + .map(ResponseEntity::ok)) + .doOnEach(sig -> audit(context, sig)); } @Override @@ -259,8 +258,8 @@ public Mono>> getConnectorPlugins( return validateAccess(context).then( Mono.just( ResponseEntity.ok( - kafkaConnectService.getConnectorPlugins(getCluster(clusterName), connectName))) - ).doOnEach(sig -> audit(context, sig)); + kafkaConnectService.getConnectorPlugins(getCluster(clusterName), connectName)))) + .doOnEach(sig -> audit(context, sig)); } @Override @@ -285,4 +284,26 @@ private Comparator getConnectorsComparator(ConnectorColumn default -> defaultComparator; }; } + + @Override + public Mono> resetConnectorOffsets(String clusterName, String connectName, + String connectorName, + ServerWebExchange exchange) { + ConnectAction[] connectActions; + + connectActions = new ConnectAction[] { ConnectAction.VIEW, ConnectAction.RESET_OFFSETS }; + + var context = AccessContext.builder() + .cluster(clusterName) + .connectActions(connectName, connectActions) + .operationName("resetConnectorOffsets") + .operationParams(Map.of(CONNECTOR_NAME, connectorName)) + .build(); + + return validateAccess(context).then( + kafkaConnectService + .resetConnectorOffsets(getCluster(clusterName), connectName, connectorName) + .map(ResponseEntity::ok)) + .doOnEach(sig -> audit(context, sig)); + } } diff --git a/api/src/main/java/io/kafbat/ui/model/rbac/permission/ConnectAction.java b/api/src/main/java/io/kafbat/ui/model/rbac/permission/ConnectAction.java index 7634e89c0..a357245bc 100644 --- a/api/src/main/java/io/kafbat/ui/model/rbac/permission/ConnectAction.java +++ b/api/src/main/java/io/kafbat/ui/model/rbac/permission/ConnectAction.java @@ -10,7 +10,8 @@ public enum ConnectAction implements PermissibleAction { EDIT(VIEW), CREATE(VIEW), RESTART(VIEW), - DELETE(VIEW) + DELETE(VIEW), + RESET_OFFSETS(VIEW) ; @@ -20,7 +21,7 @@ public enum ConnectAction implements PermissibleAction { this.dependantActions = dependantActions; } - public static final Set ALTER_ACTIONS = Set.of(CREATE, EDIT, DELETE, RESTART); + public static final Set ALTER_ACTIONS = Set.of(CREATE, EDIT, DELETE, RESTART, RESET_OFFSETS); @Nullable public static ConnectAction fromString(String name) { diff --git a/api/src/main/java/io/kafbat/ui/service/KafkaConnectService.java b/api/src/main/java/io/kafbat/ui/service/KafkaConnectService.java index f55efcddf..41b437616 100644 --- a/api/src/main/java/io/kafbat/ui/service/KafkaConnectService.java +++ b/api/src/main/java/io/kafbat/ui/service/KafkaConnectService.java @@ -52,28 +52,24 @@ public Flux getConnects(KafkaCluster cluster) { return Flux.fromIterable( Optional.ofNullable(cluster.getOriginalProperties().getKafkaConnect()) .map(lst -> lst.stream().map(clusterMapper::toKafkaConnect).toList()) - .orElse(List.of()) - ); + .orElse(List.of())); } public Flux getAllConnectors(final KafkaCluster cluster, - @Nullable final String search) { + @Nullable final String search) { return getConnects(cluster) - .flatMap(connect -> - getConnectorNamesWithErrorsSuppress(cluster, connect.getName()) - .flatMap(connectorName -> - Mono.zip( - getConnector(cluster, connect.getName(), connectorName), - getConnectorConfig(cluster, connect.getName(), connectorName), - getConnectorTasks(cluster, connect.getName(), connectorName).collectList(), - getConnectorTopics(cluster, connect.getName(), connectorName) - ).map(tuple -> - InternalConnectInfo.builder() - .connector(tuple.getT1()) - .config(tuple.getT2()) - .tasks(tuple.getT3()) - .topics(tuple.getT4().getTopics()) - .build()))) + .flatMap(connect -> getConnectorNamesWithErrorsSuppress(cluster, connect.getName()) + .flatMap(connectorName -> Mono.zip( + getConnector(cluster, connect.getName(), connectorName), + getConnectorConfig(cluster, connect.getName(), connectorName), + getConnectorTasks(cluster, connect.getName(), connectorName).collectList(), + getConnectorTopics(cluster, connect.getName(), connectorName)) + .map(tuple -> InternalConnectInfo.builder() + .connector(tuple.getT1()) + .config(tuple.getT2()) + .tasks(tuple.getT3()) + .topics(tuple.getT4().getTopics()) + .build()))) .map(kafkaConnectMapper::fullConnectorInfo) .filter(matchesSearchTerm(search)); } @@ -95,7 +91,7 @@ private Stream getStringsForSearch(FullConnectorInfoDTO fullConnectorInf } public Mono getConnectorTopics(KafkaCluster cluster, String connectClusterName, - String connectorName) { + String connectorName) { return api(cluster, connectClusterName) .mono(c -> c.getConnectorTopics(connectorName)) .map(result -> result.get(connectorName)) @@ -107,7 +103,8 @@ public Mono getConnectorTopics(KafkaCluster cluster, String con public Flux getConnectorNames(KafkaCluster cluster, String connectName) { return api(cluster, connectName) .flux(client -> client.getConnectors(null)) - // for some reason `getConnectors` method returns the response as a single string + // for some reason `getConnectors` method returns the response as a single + // string .collectList().map(e -> e.get(0)) .map(this::parseConnectorsNamesStringToList) .flatMapMany(Flux::fromIterable); @@ -125,64 +122,59 @@ private List parseConnectorsNamesStringToList(String json) { } public Mono createConnector(KafkaCluster cluster, String connectName, - Mono connector) { + Mono connector) { return api(cluster, connectName) - .mono(client -> - connector - .flatMap(c -> connectorExists(cluster, connectName, c.getName()) - .map(exists -> { - if (Boolean.TRUE.equals(exists)) { - throw new ValidationException( - String.format("Connector with name %s already exists", c.getName())); - } - return c; - })) - .map(kafkaConnectMapper::toClient) - .flatMap(client::createConnector) - .flatMap(c -> getConnector(cluster, connectName, c.getName())) - ); + .mono(client -> connector + .flatMap(c -> connectorExists(cluster, connectName, c.getName()) + .map(exists -> { + if (Boolean.TRUE.equals(exists)) { + throw new ValidationException( + String.format("Connector with name %s already exists", c.getName())); + } + return c; + })) + .map(kafkaConnectMapper::toClient) + .flatMap(client::createConnector) + .flatMap(c -> getConnector(cluster, connectName, c.getName()))); } private Mono connectorExists(KafkaCluster cluster, String connectName, - String connectorName) { + String connectorName) { return getConnectorNames(cluster, connectName) .any(name -> name.equals(connectorName)); } public Mono getConnector(KafkaCluster cluster, String connectName, - String connectorName) { + String connectorName) { return api(cluster, connectName) .mono(client -> client.getConnector(connectorName) .map(kafkaConnectMapper::fromClient) - .flatMap(connector -> - client.getConnectorStatus(connector.getName()) - // status request can return 404 if tasks not assigned yet - .onErrorResume(WebClientResponseException.NotFound.class, - e -> emptyStatus(connectorName)) - .map(connectorStatus -> { - var status = connectorStatus.getConnector(); - var sanitizedConfig = kafkaConfigSanitizer.sanitizeConnectorConfig(connector.getConfig()); - ConnectorDTO result = new ConnectorDTO() - .connect(connectName) - .status(kafkaConnectMapper.fromClient(status)) - .type(connector.getType()) - .tasks(connector.getTasks()) - .name(connector.getName()) - .config(sanitizedConfig); + .flatMap(connector -> client.getConnectorStatus(connector.getName()) + // status request can return 404 if tasks not assigned yet + .onErrorResume(WebClientResponseException.NotFound.class, + e -> emptyStatus(connectorName)) + .map(connectorStatus -> { + var status = connectorStatus.getConnector(); + var sanitizedConfig = kafkaConfigSanitizer.sanitizeConnectorConfig(connector.getConfig()); + ConnectorDTO result = new ConnectorDTO() + .connect(connectName) + .status(kafkaConnectMapper.fromClient(status)) + .type(connector.getType()) + .tasks(connector.getTasks()) + .name(connector.getName()) + .config(sanitizedConfig); - if (connectorStatus.getTasks() != null) { - boolean isAnyTaskFailed = connectorStatus.getTasks().stream() - .map(TaskStatus::getState) - .anyMatch(TaskStatus.StateEnum.FAILED::equals); + if (connectorStatus.getTasks() != null) { + boolean isAnyTaskFailed = connectorStatus.getTasks().stream() + .map(TaskStatus::getState) + .anyMatch(TaskStatus.StateEnum.FAILED::equals); - if (isAnyTaskFailed) { - result.getStatus().state(ConnectorStateDTO.TASK_FAILED); - } - } - return result; - }) - ) - ); + if (isAnyTaskFailed) { + result.getStatus().state(ConnectorStateDTO.TASK_FAILED); + } + } + return result; + }))); } private Mono emptyStatus(String connectorName) { @@ -194,19 +186,18 @@ private Mono emptyStatus(String connectorName) { } public Mono> getConnectorConfig(KafkaCluster cluster, String connectName, - String connectorName) { + String connectorName) { return api(cluster, connectName) .mono(c -> c.getConnectorConfig(connectorName)) .map(kafkaConfigSanitizer::sanitizeConnectorConfig); } public Mono setConnectorConfig(KafkaCluster cluster, String connectName, - String connectorName, Mono> requestBody) { + String connectorName, Mono> requestBody) { return api(cluster, connectName) - .mono(c -> - requestBody - .flatMap(body -> c.setConnectorConfig(connectorName, body)) - .map(kafkaConnectMapper::fromClient)); + .mono(c -> requestBody + .flatMap(body -> c.setConnectorConfig(connectorName, body)) + .map(kafkaConnectMapper::fromClient)); } public Mono deleteConnector( @@ -216,7 +207,7 @@ public Mono deleteConnector( } public Mono updateConnectorState(KafkaCluster cluster, String connectName, - String connectorName, ConnectorActionDTO action) { + String connectorName, ConnectorActionDTO action) { return api(cluster, connectName) .mono(client -> { switch (action) { @@ -240,37 +231,33 @@ public Mono updateConnectorState(KafkaCluster cluster, String connectName, } private Mono restartTasks(KafkaCluster cluster, String connectName, - String connectorName, Predicate taskFilter) { + String connectorName, Predicate taskFilter) { return getConnectorTasks(cluster, connectName, connectorName) .filter(taskFilter) - .flatMap(t -> - restartConnectorTask(cluster, connectName, connectorName, t.getId().getTask())) + .flatMap(t -> restartConnectorTask(cluster, connectName, connectorName, t.getId().getTask())) .then(); } public Flux getConnectorTasks(KafkaCluster cluster, String connectName, String connectorName) { return api(cluster, connectName) - .flux(client -> - client.getConnectorTasks(connectorName) - .onErrorResume(WebClientResponseException.NotFound.class, e -> Flux.empty()) + .flux(client -> client.getConnectorTasks(connectorName) + .onErrorResume(WebClientResponseException.NotFound.class, e -> Flux.empty()) + .map(kafkaConnectMapper::fromClient) + .flatMap(task -> client + .getConnectorTaskStatus(connectorName, task.getId().getTask()) + .onErrorResume(WebClientResponseException.NotFound.class, e -> Mono.empty()) .map(kafkaConnectMapper::fromClient) - .flatMap(task -> - client - .getConnectorTaskStatus(connectorName, task.getId().getTask()) - .onErrorResume(WebClientResponseException.NotFound.class, e -> Mono.empty()) - .map(kafkaConnectMapper::fromClient) - .map(task::status) - )); + .map(task::status))); } public Mono restartConnectorTask(KafkaCluster cluster, String connectName, - String connectorName, Integer taskId) { + String connectorName, Integer taskId) { return api(cluster, connectName) .mono(client -> client.restartConnectorTask(connectorName, taskId)); } public Flux getConnectorPlugins(KafkaCluster cluster, - String connectName) { + String connectName) { return api(cluster, connectName) .flux(client -> client.getConnectorPlugins().map(kafkaConnectMapper::fromClient)); } @@ -278,12 +265,9 @@ public Flux getConnectorPlugins(KafkaCluster cluster, public Mono validateConnectorPluginConfig( KafkaCluster cluster, String connectName, String pluginName, Mono> requestBody) { return api(cluster, connectName) - .mono(client -> - requestBody - .flatMap(body -> - client.validateConnectorPluginConfig(pluginName, body)) - .map(kafkaConnectMapper::fromClient) - ); + .mono(client -> requestBody + .flatMap(body -> client.validateConnectorPluginConfig(pluginName, body)) + .map(kafkaConnectMapper::fromClient)); } private ReactiveFailover api(KafkaCluster cluster, String connectName) { @@ -294,4 +278,10 @@ private ReactiveFailover api(KafkaCluster cluster, String } return client; } + + public Mono resetConnectorOffsets(KafkaCluster cluster, String connectName, + String connectorName) { + return api(cluster, connectName) + .mono(client -> client.resetConnectorOffsets(connectorName)); + } } diff --git a/contract/src/main/resources/swagger/kafbat-ui-api.yaml b/contract/src/main/resources/swagger/kafbat-ui-api.yaml index 6e4d8bffc..f8b217355 100644 --- a/contract/src/main/resources/swagger/kafbat-ui-api.yaml +++ b/contract/src/main/resources/swagger/kafbat-ui-api.yaml @@ -1690,6 +1690,32 @@ paths: 200: description: OK + /api/clusters/{clusterName}/connects/{connectName}/connectors/{connectorName}/offsets: + delete: + tags: + - Kafka Connect + summary: reset the offsets for the specified connector + operationId: resetConnectorOffsets + parameters: + - name: clusterName + in: path + required: true + schema: + type: string + - name: connectName + in: path + required: true + schema: + type: string + - name: connectorName + in: path + required: true + schema: + type: string + responses: + 200: + description: OK + /api/clusters/{clusterName}/ksql/v2: post: tags: diff --git a/contract/src/main/resources/swagger/kafka-connect-api.yaml b/contract/src/main/resources/swagger/kafka-connect-api.yaml index 6335025e0..39e34a0d6 100644 --- a/contract/src/main/resources/swagger/kafka-connect-api.yaml +++ b/contract/src/main/resources/swagger/kafka-connect-api.yaml @@ -144,6 +144,27 @@ paths: 500: description: Internal server error + /connectors/{connector}/offsets: + delete: + tags: + - KafkaConnectClient + summary: Reset the offsets for the specified connector + operationId: resetConnectorOffsets + parameters: + - in: path + name: connector + required: true + schema: + type: string + responses: + 200: + description: OK + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/ConnectorOffsetsReset' /connectors/{connectorName}/status: get: @@ -448,6 +469,14 @@ components: trace: type: string + ConnectorOffsetsReset: + type: object + properties: + error_code: + type: number + message: + type: string + ConnectorStatus: type: object properties: @@ -577,7 +606,5 @@ components: items: type: string - security: - basicAuth: [] - diff --git a/frontend/src/components/Connect/Details/Actions/Actions.tsx b/frontend/src/components/Connect/Details/Actions/Actions.tsx index 1948dbf04..60acff81e 100644 --- a/frontend/src/components/Connect/Details/Actions/Actions.tsx +++ b/frontend/src/components/Connect/Details/Actions/Actions.tsx @@ -11,6 +11,7 @@ import useAppParams from 'lib/hooks/useAppParams'; import { useConnector, useDeleteConnector, + useResetConnectorOffsets, useUpdateConnectorState, } from 'lib/hooks/api/kafkaConnect'; import { @@ -63,6 +64,17 @@ const Actions: React.FC = () => { stateMutation.mutateAsync(ConnectorAction.STOP); const resumeConnectorHandler = () => stateMutation.mutateAsync(ConnectorAction.RESUME); + + const resetConnectorOffsetsMutation = useResetConnectorOffsets(routerProps); + const resetConnectorOffsetsHandler = () => + confirm( + <> + Are you sure you want to reset {routerProps.connectorName}{' '} + connector offsets? + , + () => resetConnectorOffsetsMutation.mutateAsync() + ); + return ( { + {connector?.status.state === ConnectorState.STOPPED && ( + + Reset Connector Offsets + + )} > = ({ connectName: connect, connectorName: name, }); + const resetConnectorOffsetsMutation = useResetConnectorOffsets({ + clusterName, + connectName: connect, + connectorName: name, + }); const handleDelete = () => { confirm( <> - Are you sure want to remove {name} connector? + Are you sure you want to remove {name} connector? , async () => { await deleteMutation.mutateAsync(); @@ -64,6 +70,15 @@ const ActionsCell: React.FC> = ({ const stopConnectorHandler = () => stateMutation.mutateAsync(ConnectorAction.STOP); + const resetOffsetsHandler = () => { + confirm( + <> + Are you sure you want to reset the {name} connector offsets? + , + () => resetConnectorOffsetsMutation.mutateAsync() + ); + }; + return ( {(status.state === ConnectorState.PAUSED || status.state === ConnectorState.STOPPED) && ( @@ -138,6 +153,20 @@ const ActionsCell: React.FC> = ({ > Restart Failed Tasks + {status.state === ConnectorState.STOPPED && ( + + Reset Connector Offsets + + )} Remove Connector diff --git a/frontend/src/lib/hooks/api/kafkaConnect.ts b/frontend/src/lib/hooks/api/kafkaConnect.ts index 225e72165..4d79d358a 100644 --- a/frontend/src/lib/hooks/api/kafkaConnect.ts +++ b/frontend/src/lib/hooks/api/kafkaConnect.ts @@ -161,3 +161,11 @@ export function useDeleteConnector(props: UseConnectorProps) { onSuccess: () => client.invalidateQueries(connectorsKey(props.clusterName)), }); } + +export function useResetConnectorOffsets(props: UseConnectorProps) { + const client = useQueryClient(); + + return useMutation(() => api.resetConnectorOffsets(props), { + onSuccess: () => client.invalidateQueries(connectorKey(props)), + }); +} From b8190136129c6c16dda49077cc7539c2811045d7 Mon Sep 17 00:00:00 2001 From: NOZAIS Julien Date: Tue, 1 Oct 2024 18:30:38 +0200 Subject: [PATCH 05/34] Disable the "Remove Connector" button if the action is not allowed for the user --- frontend/src/components/Connect/List/ActionsCell.tsx | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/frontend/src/components/Connect/List/ActionsCell.tsx b/frontend/src/components/Connect/List/ActionsCell.tsx index 0bc591477..3f8dcd0e5 100644 --- a/frontend/src/components/Connect/List/ActionsCell.tsx +++ b/frontend/src/components/Connect/List/ActionsCell.tsx @@ -167,9 +167,16 @@ const ActionsCell: React.FC> = ({ Reset Connector Offsets )} - + Remove Connector - + ); }; From 26e075d9a1f357be1624d7ed931dda5247304949 Mon Sep 17 00:00:00 2001 From: NOZAIS Julien Date: Wed, 2 Oct 2024 16:59:32 +0200 Subject: [PATCH 06/34] Disable reset offsets button when connector is not stopped --- .../Connect/Details/Actions/Actions.tsx | 28 +++++++-------- .../components/Connect/List/ActionsCell.tsx | 34 +++++++++---------- 2 files changed, 29 insertions(+), 33 deletions(-) diff --git a/frontend/src/components/Connect/Details/Actions/Actions.tsx b/frontend/src/components/Connect/Details/Actions/Actions.tsx index 60acff81e..5decdb1a3 100644 --- a/frontend/src/components/Connect/Details/Actions/Actions.tsx +++ b/frontend/src/components/Connect/Details/Actions/Actions.tsx @@ -108,7 +108,7 @@ const Actions: React.FC = () => { value: routerProps.connectName, }} > - Stop Connector + Stop )} {(connector?.status.state === ConnectorState.PAUSED || connector?.status.state === ConnectorState.STOPPED) && ( @@ -159,20 +159,18 @@ const Actions: React.FC = () => { - {connector?.status.state === ConnectorState.STOPPED && ( - - Reset Connector Offsets - - )} + + Reset Offsets + > = ({ value: connect, }} > - Pause Connector + Pause )} {status.state === ConnectorState.RUNNING && ( @@ -117,7 +117,7 @@ const ActionsCell: React.FC> = ({ value: connect, }} > - Stop Connector + Stop )} > = ({ > Restart Failed Tasks - {status.state === ConnectorState.STOPPED && ( - - Reset Connector Offsets - - )} + + Reset Offsets + > = ({ action: Action.DELETE, value: connect, }}> - Remove Connector + Delete ); From d9d269bb26132879cc85baa8c5492c6489060515 Mon Sep 17 00:00:00 2001 From: NOZAIS Julien Date: Fri, 4 Oct 2024 16:04:47 +0200 Subject: [PATCH 07/34] Add frontend tests of connector reset --- .../Actions/__tests__/Actions.spec.tsx | 41 ++++++++++++++++++- .../Connect/List/__tests__/List.spec.tsx | 12 +++--- 2 files changed, 46 insertions(+), 7 deletions(-) diff --git a/frontend/src/components/Connect/Details/Actions/__tests__/Actions.spec.tsx b/frontend/src/components/Connect/Details/Actions/__tests__/Actions.spec.tsx index 27743949b..693b90bf0 100644 --- a/frontend/src/components/Connect/Details/Actions/__tests__/Actions.spec.tsx +++ b/frontend/src/components/Connect/Details/Actions/__tests__/Actions.spec.tsx @@ -40,6 +40,7 @@ const expectActionButtonsExists = () => { expect(screen.getByText('Restart Connector')).toBeInTheDocument(); expect(screen.getByText('Restart All Tasks')).toBeInTheDocument(); expect(screen.getByText('Restart Failed Tasks')).toBeInTheDocument(); + expect(screen.getByText('Reset Offsets')).toBeInTheDocument(); expect(screen.getByText('Delete')).toBeInTheDocument(); }; const afterClickDropDownButton = async () => { @@ -82,6 +83,7 @@ describe('Actions', () => { expect(screen.getAllByRole('menuitem').length).toEqual(4); expect(screen.getByText('Resume')).toBeInTheDocument(); expect(screen.queryByText('Pause')).not.toBeInTheDocument(); + expect(screen.queryByText('Stop')).not.toBeInTheDocument(); expectActionButtonsExists(); }); @@ -94,6 +96,7 @@ describe('Actions', () => { expect(screen.getAllByRole('menuitem').length).toEqual(3); expect(screen.queryByText('Resume')).not.toBeInTheDocument(); expect(screen.queryByText('Pause')).not.toBeInTheDocument(); + expect(screen.queryByText('Stop')).not.toBeInTheDocument(); expectActionButtonsExists(); }); @@ -106,6 +109,7 @@ describe('Actions', () => { expect(screen.getAllByRole('menuitem').length).toEqual(3); expect(screen.queryByText('Resume')).not.toBeInTheDocument(); expect(screen.queryByText('Pause')).not.toBeInTheDocument(); + expect(screen.queryByText('Stop')).not.toBeInTheDocument(); expectActionButtonsExists(); }); @@ -118,6 +122,7 @@ describe('Actions', () => { expect(screen.getAllByRole('menuitem').length).toEqual(4); expect(screen.queryByText('Resume')).not.toBeInTheDocument(); expect(screen.getByText('Pause')).toBeInTheDocument(); + expect(screen.getByText('Stop')).toBeInTheDocument(); expectActionButtonsExists(); }); @@ -137,6 +142,15 @@ describe('Actions', () => { expect(screen.getByRole('dialog')).toBeInTheDocument(); }); + it('opens confirmation modal when reset offsets button clicked', async () => { + renderComponent(); + await afterClickDropDownButton(); + await waitFor(async () => + userEvent.click(screen.getByRole('menuitem', { name: 'Reset Offsets' })) + ); + expect(screen.getByRole('dialog')).toBeInTheDocument(); + }); + it('calls restartConnector when restart button clicked', async () => { const restartConnector = jest.fn(); (useUpdateConnectorState as jest.Mock).mockImplementation(() => ({ @@ -191,7 +205,18 @@ describe('Actions', () => { expect(pauseConnector).toHaveBeenCalledWith(ConnectorAction.PAUSE); }); - it('calls resumeConnector when resume button clicked', async () => { + it('calls stopConnector when stop button clicked', async () => { + const stopConnector = jest.fn(); + (useUpdateConnectorState as jest.Mock).mockImplementation(() => ({ + mutateAsync: stopConnector, + })); + renderComponent(); + await afterClickRestartButton(); + await userEvent.click(screen.getByRole('menuitem', { name: 'Stop' })); + expect(stopConnector).toHaveBeenCalledWith(ConnectorAction.STOP); + }); + + it('calls resumeConnector when resume button clicked from PAUSED state', async () => { const resumeConnector = jest.fn(); (useConnector as jest.Mock).mockImplementation(() => ({ data: setConnectorStatus(connector, ConnectorState.PAUSED), @@ -204,6 +229,20 @@ describe('Actions', () => { await userEvent.click(screen.getByRole('menuitem', { name: 'Resume' })); expect(resumeConnector).toHaveBeenCalledWith(ConnectorAction.RESUME); }); + + it('calls resumeConnector when resume button clicked from STOPPED state', async () => { + const resumeConnector = jest.fn(); + (useConnector as jest.Mock).mockImplementation(() => ({ + data: setConnectorStatus(connector, ConnectorState.STOPPED), + })); + (useUpdateConnectorState as jest.Mock).mockImplementation(() => ({ + mutateAsync: resumeConnector, + })); + renderComponent(); + await afterClickRestartButton(); + await userEvent.click(screen.getByRole('menuitem', { name: 'Resume' })); + expect(resumeConnector).toHaveBeenCalledWith(ConnectorAction.RESUME); + }); }); }); }); diff --git a/frontend/src/components/Connect/List/__tests__/List.spec.tsx b/frontend/src/components/Connect/List/__tests__/List.spec.tsx index 82b4aab21..8c05ad5c0 100644 --- a/frontend/src/components/Connect/List/__tests__/List.spec.tsx +++ b/frontend/src/components/Connect/List/__tests__/List.spec.tsx @@ -94,7 +94,7 @@ describe('Connectors List', () => { }); }); - describe('when remove connector modal is open', () => { + describe('when delete modal is open', () => { beforeEach(() => { (useConnectors as jest.Mock).mockImplementation(() => ({ data: connectors, @@ -104,10 +104,10 @@ describe('Connectors List', () => { })); }); - it('calls removeConnector on confirm', async () => { + it('calls deleteConnector on confirm', async () => { renderComponent(); - const removeButton = screen.getAllByText('Remove Connector')[0]; - await waitFor(() => userEvent.click(removeButton)); + const deleteButton = screen.getAllByText('Delete')[0]; + await waitFor(() => userEvent.click(deleteButton)); const submitButton = screen.getAllByRole('button', { name: 'Confirm', @@ -118,8 +118,8 @@ describe('Connectors List', () => { it('closes the modal when cancel button is clicked', async () => { renderComponent(); - const removeButton = screen.getAllByText('Remove Connector')[0]; - await waitFor(() => userEvent.click(removeButton)); + const deleteButton = screen.getAllByText('Delete')[0]; + await waitFor(() => userEvent.click(deleteButton)); const cancelButton = screen.getAllByRole('button', { name: 'Cancel', From 971c2c99277dd0ecfcbd9213399c3e9b1a5f0aef Mon Sep 17 00:00:00 2001 From: NOZAIS Julien Date: Fri, 4 Oct 2024 17:47:10 +0200 Subject: [PATCH 08/34] Add frontend tests for the reset connector offsets button --- .../Actions/__tests__/Actions.spec.tsx | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/frontend/src/components/Connect/Details/Actions/__tests__/Actions.spec.tsx b/frontend/src/components/Connect/Details/Actions/__tests__/Actions.spec.tsx index 693b90bf0..4de6c6335 100644 --- a/frontend/src/components/Connect/Details/Actions/__tests__/Actions.spec.tsx +++ b/frontend/src/components/Connect/Details/Actions/__tests__/Actions.spec.tsx @@ -84,6 +84,23 @@ describe('Actions', () => { expect(screen.getByText('Resume')).toBeInTheDocument(); expect(screen.queryByText('Pause')).not.toBeInTheDocument(); expect(screen.queryByText('Stop')).not.toBeInTheDocument(); + expect(screen.queryByText('Reset Connector Offsets')).toBeInTheDocument(); + expect(screen.getByText('Reset Connector Offsets')).toBeDisabled(); + expectActionButtonsExists(); + }); + + it('renders buttons when stopped', async () => { + (useConnector as jest.Mock).mockImplementation(() => ({ + data: setConnectorStatus(connector, ConnectorState.PAUSED), + })); + renderComponent(); + await afterClickRestartButton(); + expect(screen.getAllByRole('menuitem').length).toEqual(4); + expect(screen.getByText('Resume')).toBeInTheDocument(); + expect(screen.queryByText('Pause')).not.toBeInTheDocument(); + expect(screen.queryByText('Stop')).not.toBeInTheDocument(); + expect(screen.queryByText('Reset Connector Offsets')).toBeInTheDocument(); + expect(screen.getByText('Reset Connector Offsets')).not.toBeDisabled(); expectActionButtonsExists(); }); @@ -97,6 +114,8 @@ describe('Actions', () => { expect(screen.queryByText('Resume')).not.toBeInTheDocument(); expect(screen.queryByText('Pause')).not.toBeInTheDocument(); expect(screen.queryByText('Stop')).not.toBeInTheDocument(); + expect(screen.queryByText('Reset Connector Offsets')).toBeInTheDocument(); + expect(screen.getByText('Reset Connector Offsets')).toBeDisabled(); expectActionButtonsExists(); }); @@ -110,6 +129,8 @@ describe('Actions', () => { expect(screen.queryByText('Resume')).not.toBeInTheDocument(); expect(screen.queryByText('Pause')).not.toBeInTheDocument(); expect(screen.queryByText('Stop')).not.toBeInTheDocument(); + expect(screen.queryByText('Reset Connector Offsets')).toBeInTheDocument(); + expect(screen.getByText('Reset Connector Offsets')).toBeDisabled(); expectActionButtonsExists(); }); @@ -123,6 +144,8 @@ describe('Actions', () => { expect(screen.queryByText('Resume')).not.toBeInTheDocument(); expect(screen.getByText('Pause')).toBeInTheDocument(); expect(screen.getByText('Stop')).toBeInTheDocument(); + expect(screen.queryByText('Reset Connector Offsets')).toBeInTheDocument(); + expect(screen.getByText('Reset Connector Offsets')).toBeDisabled(); expectActionButtonsExists(); }); From c9d8aad931408d1fd27c4063b29a08a303edea19 Mon Sep 17 00:00:00 2001 From: NOZAIS Julien Date: Fri, 4 Oct 2024 18:21:30 +0200 Subject: [PATCH 09/34] Fix code formatting and typos --- .../ui/client/RetryingKafkaConnectClient.java | 45 ++++++++----------- .../ui/controller/KafkaConnectController.java | 30 ++++++++----- .../Connect/Details/Actions/Actions.tsx | 7 ++- .../Actions/__tests__/Actions.spec.tsx | 24 +++++----- .../components/Connect/List/ActionsCell.tsx | 6 ++- 5 files changed, 61 insertions(+), 51 deletions(-) diff --git a/api/src/main/java/io/kafbat/ui/client/RetryingKafkaConnectClient.java b/api/src/main/java/io/kafbat/ui/client/RetryingKafkaConnectClient.java index 2a13cb961..eeac1f1e9 100644 --- a/api/src/main/java/io/kafbat/ui/client/RetryingKafkaConnectClient.java +++ b/api/src/main/java/io/kafbat/ui/client/RetryingKafkaConnectClient.java @@ -34,8 +34,8 @@ public class RetryingKafkaConnectClient extends KafkaConnectClientApi { private static final Duration RETRIES_DELAY = Duration.ofMillis(200); public RetryingKafkaConnectClient(ClustersProperties.ConnectCluster config, - @Nullable ClustersProperties.TruststoreConfig truststoreConfig, - DataSize maxBuffSize) { + @Nullable ClustersProperties.TruststoreConfig truststoreConfig, + DataSize maxBuffSize) { super(new RetryingApiClient(config, truststoreConfig, maxBuffSize)); } @@ -43,9 +43,8 @@ private static Retry conflictCodeRetry() { return Retry .fixedDelay(MAX_RETRIES, RETRIES_DELAY) .filter(e -> e instanceof WebClientResponseException.Conflict) - .onRetryExhaustedThrow((spec, signal) -> - new KafkaConnectConflictReponseException( - (WebClientResponseException.Conflict) signal.failure())); + .onRetryExhaustedThrow((spec, signal) -> new KafkaConnectConflictReponseException( + (WebClientResponseException.Conflict) signal.failure())); } private static Mono withRetryOnConflict(Mono publisher) { @@ -58,25 +57,23 @@ private static Flux withRetryOnConflict(Flux publisher) { private static Mono withBadRequestErrorHandling(Mono publisher) { return publisher - .onErrorResume(WebClientResponseException.BadRequest.class, e -> - Mono.error(new ValidationException("Invalid configuration"))) - .onErrorResume(WebClientResponseException.InternalServerError.class, e -> - Mono.error(new ValidationException("Invalid configuration"))); + .onErrorResume(WebClientResponseException.BadRequest.class, + e -> Mono.error(new ValidationException("Invalid configuration"))) + .onErrorResume(WebClientResponseException.InternalServerError.class, + e -> Mono.error(new ValidationException("Invalid configuration"))); } @Override public Mono createConnector(NewConnector newConnector) throws RestClientException { return withBadRequestErrorHandling( - super.createConnector(newConnector) - ); + super.createConnector(newConnector)); } @Override public Mono setConnectorConfig(String connectorName, Map requestBody) throws RestClientException { return withBadRequestErrorHandling( - super.setConnectorConfig(connectorName, requestBody) - ); + super.setConnectorConfig(connectorName, requestBody)); } @Override @@ -96,7 +93,6 @@ public Mono> deleteConnectorWithHttpInfo(String connectorNa return withRetryOnConflict(super.deleteConnectorWithHttpInfo(connectorName)); } - @Override public Mono getConnector(String connectorName) throws WebClientResponseException { return withRetryOnConflict(super.getConnector(connectorName)); @@ -208,7 +204,7 @@ public Mono restartConnector(String connectorName, Boolean includeTasks, B @Override public Mono> restartConnectorWithHttpInfo(String connectorName, Boolean includeTasks, - Boolean onlyFailed) throws WebClientResponseException { + Boolean onlyFailed) throws WebClientResponseException { return withRetryOnConflict(super.restartConnectorWithHttpInfo(connectorName, includeTasks, onlyFailed)); } @@ -236,14 +232,14 @@ public Mono> resumeConnectorWithHttpInfo(String connectorNa @Override public Mono> setConnectorConfigWithHttpInfo(String connectorName, - Map requestBody) + Map requestBody) throws WebClientResponseException { return withRetryOnConflict(super.setConnectorConfigWithHttpInfo(connectorName, requestBody)); } @Override public Mono validateConnectorPluginConfig(String pluginName, - Map requestBody) + Map requestBody) throws WebClientResponseException { return withRetryOnConflict(super.validateConnectorPluginConfig(pluginName, requestBody)); } @@ -257,8 +253,8 @@ public Mono> validateCon private static class RetryingApiClient extends ApiClient { public RetryingApiClient(ClustersProperties.ConnectCluster config, - ClustersProperties.TruststoreConfig truststoreConfig, - DataSize maxBuffSize) { + ClustersProperties.TruststoreConfig truststoreConfig, + DataSize maxBuffSize) { super(buildWebClient(maxBuffSize, config, truststoreConfig), null, null); setBasePath(config.getAddress()); setUsername(config.getUsername()); @@ -266,20 +262,17 @@ public RetryingApiClient(ClustersProperties.ConnectCluster config, } public static WebClient buildWebClient(DataSize maxBuffSize, - ClustersProperties.ConnectCluster config, - ClustersProperties.TruststoreConfig truststoreConfig) { + ClustersProperties.ConnectCluster config, + ClustersProperties.TruststoreConfig truststoreConfig) { return new WebClientConfigurator() .configureSsl( truststoreConfig, new ClustersProperties.KeystoreConfig( config.getKeystoreLocation(), - config.getKeystorePassword() - ) - ) + config.getKeystorePassword())) .configureBasicAuth( config.getUsername(), - config.getPassword() - ) + config.getPassword()) .configureBufferSize(maxBuffSize) .build(); } diff --git a/api/src/main/java/io/kafbat/ui/controller/KafkaConnectController.java b/api/src/main/java/io/kafbat/ui/controller/KafkaConnectController.java index fd6838ab7..e992d4d46 100644 --- a/api/src/main/java/io/kafbat/ui/controller/KafkaConnectController.java +++ b/api/src/main/java/io/kafbat/ui/controller/KafkaConnectController.java @@ -62,7 +62,8 @@ public Mono>> getConnectors(String clusterName, Stri return validateAccess(context) .thenReturn( - ResponseEntity.ok(kafkaConnectService.getConnectorNames(getCluster(clusterName), connectName))) + ResponseEntity.ok(kafkaConnectService.getConnectorNames( + getCluster(clusterName), connectName))) .doOnEach(sig -> audit(context, sig)); } @@ -135,7 +136,8 @@ public Mono>> getAllConnectors( : getConnectorsComparator(orderBy).reversed(); Flux job = kafkaConnectService.getAllConnectors(getCluster(clusterName), search) - .filterWhen(dto -> accessControlService.isConnectAccessible(dto.getConnect(), clusterName)) + .filterWhen(dto -> accessControlService.isConnectAccessible(dto.getConnect(), + clusterName)) .sort(comparator); return Mono.just(ResponseEntity.ok(job)) @@ -176,7 +178,8 @@ public Mono> setConnectorConfig(String clusterName, return validateAccess(context).then( kafkaConnectService - .setConnectorConfig(getCluster(clusterName), connectName, connectorName, requestBody) + .setConnectorConfig(getCluster(clusterName), connectName, connectorName, + requestBody) .map(ResponseEntity::ok)) .doOnEach(sig -> audit(context, sig)); } @@ -202,7 +205,8 @@ public Mono> updateConnectorState(String clusterName, Strin return validateAccess(context).then( kafkaConnectService - .updateConnectorState(getCluster(clusterName), connectName, connectorName, action) + .updateConnectorState(getCluster(clusterName), connectName, + connectorName, action) .map(ResponseEntity::ok)) .doOnEach(sig -> audit(context, sig)); } @@ -222,7 +226,8 @@ public Mono>> getConnectorTasks(String clusterName, return validateAccess(context).thenReturn( ResponseEntity .ok(kafkaConnectService - .getConnectorTasks(getCluster(clusterName), connectName, connectorName))) + .getConnectorTasks(getCluster(clusterName), connectName, + connectorName))) .doOnEach(sig -> audit(context, sig)); } @@ -240,7 +245,8 @@ public Mono> restartConnectorTask(String clusterName, Strin return validateAccess(context).then( kafkaConnectService - .restartConnectorTask(getCluster(clusterName), connectName, connectorName, taskId) + .restartConnectorTask(getCluster(clusterName), connectName, + connectorName, taskId) .map(ResponseEntity::ok)) .doOnEach(sig -> audit(context, sig)); } @@ -258,13 +264,15 @@ public Mono>> getConnectorPlugins( return validateAccess(context).then( Mono.just( ResponseEntity.ok( - kafkaConnectService.getConnectorPlugins(getCluster(clusterName), connectName)))) + kafkaConnectService.getConnectorPlugins( + getCluster(clusterName), connectName)))) .doOnEach(sig -> audit(context, sig)); } @Override public Mono> validateConnectorPluginConfig( - String clusterName, String connectName, String pluginName, @Valid Mono> requestBody, + String clusterName, String connectName, String pluginName, + @Valid Mono> requestBody, ServerWebExchange exchange) { return kafkaConnectService .validateConnectorPluginConfig( @@ -280,7 +288,8 @@ private Comparator getConnectorsComparator(ConnectorColumn return switch (orderBy) { case CONNECT -> Comparator.comparing(FullConnectorInfoDTO::getConnect); case TYPE -> Comparator.comparing(FullConnectorInfoDTO::getType); - case STATUS -> Comparator.comparing(fullConnectorInfoDTO -> fullConnectorInfoDTO.getStatus().getState()); + case STATUS -> Comparator + .comparing(fullConnectorInfoDTO -> fullConnectorInfoDTO.getStatus().getState()); default -> defaultComparator; }; } @@ -302,7 +311,8 @@ public Mono> resetConnectorOffsets(String clusterName, Stri return validateAccess(context).then( kafkaConnectService - .resetConnectorOffsets(getCluster(clusterName), connectName, connectorName) + .resetConnectorOffsets(getCluster(clusterName), connectName, + connectorName) .map(ResponseEntity::ok)) .doOnEach(sig -> audit(context, sig)); } diff --git a/frontend/src/components/Connect/Details/Actions/Actions.tsx b/frontend/src/components/Connect/Details/Actions/Actions.tsx index 5decdb1a3..ef70c3a30 100644 --- a/frontend/src/components/Connect/Details/Actions/Actions.tsx +++ b/frontend/src/components/Connect/Details/Actions/Actions.tsx @@ -111,7 +111,8 @@ const Actions: React.FC = () => { Stop )} - {(connector?.status.state === ConnectorState.PAUSED || connector?.status.state === ConnectorState.STOPPED) && ( + {(connector?.status.state === ConnectorState.PAUSED || + connector?.status.state === ConnectorState.STOPPED) && ( { { expect(screen.getByText('Resume')).toBeInTheDocument(); expect(screen.queryByText('Pause')).not.toBeInTheDocument(); expect(screen.queryByText('Stop')).not.toBeInTheDocument(); - expect(screen.queryByText('Reset Connector Offsets')).toBeInTheDocument(); - expect(screen.getByText('Reset Connector Offsets')).toBeDisabled(); + expect(screen.queryByText('Reset Offsets')).toBeInTheDocument(); + expect(screen.getByText('Reset Offsets')).toBeDisabled(); expectActionButtonsExists(); }); @@ -99,8 +99,8 @@ describe('Actions', () => { expect(screen.getByText('Resume')).toBeInTheDocument(); expect(screen.queryByText('Pause')).not.toBeInTheDocument(); expect(screen.queryByText('Stop')).not.toBeInTheDocument(); - expect(screen.queryByText('Reset Connector Offsets')).toBeInTheDocument(); - expect(screen.getByText('Reset Connector Offsets')).not.toBeDisabled(); + expect(screen.queryByText('Reset Offsets')).toBeInTheDocument(); + expect(screen.getByText('Reset Offsets')).toBeEnabled(); expectActionButtonsExists(); }); @@ -114,8 +114,8 @@ describe('Actions', () => { expect(screen.queryByText('Resume')).not.toBeInTheDocument(); expect(screen.queryByText('Pause')).not.toBeInTheDocument(); expect(screen.queryByText('Stop')).not.toBeInTheDocument(); - expect(screen.queryByText('Reset Connector Offsets')).toBeInTheDocument(); - expect(screen.getByText('Reset Connector Offsets')).toBeDisabled(); + expect(screen.queryByText('Reset Offsets')).toBeInTheDocument(); + expect(screen.getByText('Reset Offsets')).toBeDisabled(); expectActionButtonsExists(); }); @@ -129,8 +129,8 @@ describe('Actions', () => { expect(screen.queryByText('Resume')).not.toBeInTheDocument(); expect(screen.queryByText('Pause')).not.toBeInTheDocument(); expect(screen.queryByText('Stop')).not.toBeInTheDocument(); - expect(screen.queryByText('Reset Connector Offsets')).toBeInTheDocument(); - expect(screen.getByText('Reset Connector Offsets')).toBeDisabled(); + expect(screen.queryByText('Reset Offsets')).toBeInTheDocument(); + expect(screen.getByText('Reset Offsets')).toBeDisabled(); expectActionButtonsExists(); }); @@ -144,8 +144,8 @@ describe('Actions', () => { expect(screen.queryByText('Resume')).not.toBeInTheDocument(); expect(screen.getByText('Pause')).toBeInTheDocument(); expect(screen.getByText('Stop')).toBeInTheDocument(); - expect(screen.queryByText('Reset Connector Offsets')).toBeInTheDocument(); - expect(screen.getByText('Reset Connector Offsets')).toBeDisabled(); + expect(screen.queryByText('Reset Offsets')).toBeInTheDocument(); + expect(screen.getByText('Reset Offsets')).toBeDisabled(); expectActionButtonsExists(); }); @@ -169,7 +169,9 @@ describe('Actions', () => { renderComponent(); await afterClickDropDownButton(); await waitFor(async () => - userEvent.click(screen.getByRole('menuitem', { name: 'Reset Offsets' })) + userEvent.click( + screen.getByRole('menuitem', { name: 'Reset Offsets' }) + ) ); expect(screen.getByRole('dialog')).toBeInTheDocument(); }); diff --git a/frontend/src/components/Connect/List/ActionsCell.tsx b/frontend/src/components/Connect/List/ActionsCell.tsx index 6d1c3e146..eb2c4699e 100644 --- a/frontend/src/components/Connect/List/ActionsCell.tsx +++ b/frontend/src/components/Connect/List/ActionsCell.tsx @@ -81,7 +81,8 @@ const ActionsCell: React.FC> = ({ return ( - {(status.state === ConnectorState.PAUSED || status.state === ConnectorState.STOPPED) && ( + {(status.state === ConnectorState.PAUSED || + status.state === ConnectorState.STOPPED) && ( > = ({ resource: ResourceType.CONNECT, action: Action.DELETE, value: connect, - }}> + }} + > Delete From a36bc2012fc35b27a743c2f740c34d29abc13aac Mon Sep 17 00:00:00 2001 From: NOZAIS Julien Date: Fri, 4 Oct 2024 18:28:23 +0200 Subject: [PATCH 10/34] Fix the confirmation message --- frontend/src/components/Connect/Details/Actions/Actions.tsx | 4 ++-- frontend/src/components/Connect/List/ActionsCell.tsx | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/frontend/src/components/Connect/Details/Actions/Actions.tsx b/frontend/src/components/Connect/Details/Actions/Actions.tsx index ef70c3a30..379384ab8 100644 --- a/frontend/src/components/Connect/Details/Actions/Actions.tsx +++ b/frontend/src/components/Connect/Details/Actions/Actions.tsx @@ -38,7 +38,7 @@ const Actions: React.FC = () => { const deleteConnectorHandler = () => confirm( <> - Are you sure you want to remove {routerProps.connectorName}{' '} + Are you sure you want to remove the {routerProps.connectorName}{' '} connector? , async () => { @@ -69,7 +69,7 @@ const Actions: React.FC = () => { const resetConnectorOffsetsHandler = () => confirm( <> - Are you sure you want to reset {routerProps.connectorName}{' '} + Are you sure you want to reset the {routerProps.connectorName}{' '} connector offsets? , () => resetConnectorOffsetsMutation.mutateAsync() diff --git a/frontend/src/components/Connect/List/ActionsCell.tsx b/frontend/src/components/Connect/List/ActionsCell.tsx index eb2c4699e..4ad50c00e 100644 --- a/frontend/src/components/Connect/List/ActionsCell.tsx +++ b/frontend/src/components/Connect/List/ActionsCell.tsx @@ -45,7 +45,7 @@ const ActionsCell: React.FC> = ({ const handleDelete = () => { confirm( <> - Are you sure you want to remove {name} connector? + Are you sure you want to remove the {name} connector? , async () => { await deleteMutation.mutateAsync(); From b21d04e85cd6c0933ee030ab7646df1657a100a9 Mon Sep 17 00:00:00 2001 From: NOZAIS Julien Date: Fri, 4 Oct 2024 18:45:21 +0200 Subject: [PATCH 11/34] fix missing resetConnectorOffsets mock in frontend unit tests --- .../Connect/Details/Actions/__tests__/Actions.spec.tsx | 3 +++ frontend/src/components/Connect/List/__tests__/List.spec.tsx | 1 + 2 files changed, 4 insertions(+) diff --git a/frontend/src/components/Connect/Details/Actions/__tests__/Actions.spec.tsx b/frontend/src/components/Connect/Details/Actions/__tests__/Actions.spec.tsx index 87b2a3ffb..34c574008 100644 --- a/frontend/src/components/Connect/Details/Actions/__tests__/Actions.spec.tsx +++ b/frontend/src/components/Connect/Details/Actions/__tests__/Actions.spec.tsx @@ -23,6 +23,7 @@ function setConnectorStatus(con: Connector, state: ConnectorState) { const mockHistoryPush = jest.fn(); const deleteConnector = jest.fn(); +const resetConnectorOffsets = jest.fn(); const cancelMock = jest.fn(); jest.mock('react-router-dom', () => ({ @@ -34,6 +35,7 @@ jest.mock('lib/hooks/api/kafkaConnect', () => ({ useConnector: jest.fn(), useDeleteConnector: jest.fn(), useUpdateConnectorState: jest.fn(), + useResetConnectorOffsets: jest.fn(), })); const expectActionButtonsExists = () => { @@ -56,6 +58,7 @@ describe('Actions', () => { mockHistoryPush.mockClear(); deleteConnector.mockClear(); cancelMock.mockClear(); + resetConnectorOffsets.mockClear(); }); describe('view', () => { diff --git a/frontend/src/components/Connect/List/__tests__/List.spec.tsx b/frontend/src/components/Connect/List/__tests__/List.spec.tsx index 8c05ad5c0..3e86448bc 100644 --- a/frontend/src/components/Connect/List/__tests__/List.spec.tsx +++ b/frontend/src/components/Connect/List/__tests__/List.spec.tsx @@ -27,6 +27,7 @@ jest.mock('lib/hooks/api/kafkaConnect', () => ({ useConnectors: jest.fn(), useDeleteConnector: jest.fn(), useUpdateConnectorState: jest.fn(), + useResetConnectorOffsets: jest.fn(), })); const clusterName = 'local'; From 0c3495df5835180a7f158641e1a490d5ad39b4a4 Mon Sep 17 00:00:00 2001 From: NOZAIS Julien Date: Mon, 7 Oct 2024 10:40:47 +0200 Subject: [PATCH 12/34] Revert "format kafkbat-ui openapi definition" This reverts commit 6ce7078533124de509f7c92b3470f1b075911110. --- .../main/resources/swagger/kafbat-ui-api.yaml | 94 +++++++++++-------- 1 file changed, 57 insertions(+), 37 deletions(-) diff --git a/contract/src/main/resources/swagger/kafbat-ui-api.yaml b/contract/src/main/resources/swagger/kafbat-ui-api.yaml index f8b217355..45d560d9a 100644 --- a/contract/src/main/resources/swagger/kafbat-ui-api.yaml +++ b/contract/src/main/resources/swagger/kafbat-ui-api.yaml @@ -4,7 +4,7 @@ info: version: 0.1.0 title: Api Documentation termsOfService: urn:tos - contact: {} + contact: { } license: name: Apache 2.0 url: https://www.apache.org/licenses/LICENSE-2.0 @@ -31,6 +31,7 @@ paths: items: $ref: '#/components/schemas/Cluster' + /api/clusters/{clusterName}/cache: post: tags: @@ -53,6 +54,7 @@ paths: 404: description: Not found + /api/clusters/{clusterName}/brokers: get: tags: @@ -430,6 +432,7 @@ paths: 404: description: Not found + /api/clusters/{clusterName}/topics/{topicName}: get: tags: @@ -641,6 +644,7 @@ paths: schema: $ref: '#/components/schemas/SmartFilterTestExecutionResult' + /api/clusters/{clusterName}/topics/{topicName}/messages: get: tags: @@ -661,7 +665,7 @@ paths: - name: seekType in: query schema: - $ref: '#/components/schemas/SeekType' + $ref: "#/components/schemas/SeekType" - name: seekTo in: query schema: @@ -680,19 +684,19 @@ paths: - name: filterQueryType in: query schema: - $ref: '#/components/schemas/MessageFilterType' + $ref: "#/components/schemas/MessageFilterType" - name: seekDirection in: query schema: - $ref: '#/components/schemas/SeekDirection' + $ref: "#/components/schemas/SeekDirection" - name: keySerde in: query - description: 'Serde that should be used for deserialization. Will be chosen automatically if not set.' + description: "Serde that should be used for deserialization. Will be chosen automatically if not set." schema: type: string - name: valueSerde in: query - description: 'Serde that should be used for deserialization. Will be chosen automatically if not set.' + description: "Serde that should be used for deserialization. Will be chosen automatically if not set." schema: type: string responses: @@ -789,6 +793,7 @@ paths: schema: $ref: '#/components/schemas/MessageFilterId' + /api/clusters/{clusterName}/topics/{topicName}/messages/v2: get: tags: @@ -810,7 +815,7 @@ paths: in: query description: Messages polling mode schema: - $ref: '#/components/schemas/PollingMode' + $ref: "#/components/schemas/PollingMode" - name: partitions in: query schema: @@ -847,17 +852,17 @@ paths: format: int64 - name: keySerde in: query - description: 'Serde that should be used for deserialization. Will be chosen automatically if not set.' + description: "Serde that should be used for deserialization. Will be chosen automatically if not set." schema: type: string - name: valueSerde in: query - description: 'Serde that should be used for deserialization. Will be chosen automatically if not set.' + description: "Serde that should be used for deserialization. Will be chosen automatically if not set." schema: type: string - name: cursor in: query - description: 'id of the cursor for pagination, if passed - all other query params ignored' + description: "id of the cursor for pagination, if passed - all other query params ignored" schema: type: string responses: @@ -870,6 +875,7 @@ paths: items: $ref: '#/components/schemas/TopicMessageEvent' + /api/clusters/{clusterName}/topics/{topicName}/activeproducers: get: tags: @@ -897,6 +903,7 @@ paths: items: $ref: '#/components/schemas/TopicProducerState' + /api/clusters/{clusterName}/topics/{topicName}/consumer-groups: get: tags: @@ -969,6 +976,7 @@ paths: schema: $ref: '#/components/schemas/ConsumerGroupsPageResponse' + /api/clusters/{clusterName}/consumer-groups/{id}: get: tags: @@ -1200,6 +1208,7 @@ paths: 404: description: Not found + /api/clusters/{clusterName}/schemas/{subject}/versions/{version}: get: tags: @@ -2210,6 +2219,7 @@ paths: schema: $ref: '#/components/schemas/ApplicationConfigValidation' + /api/config/relatedfiles: post: tags: @@ -2255,7 +2265,7 @@ components: description: type: string preferred: - description: 'This serde was automatically chosen by cluster config. This should be enabled in UI by default. Also it will be used for deserialization if no serdes passed.' + description: "This serde was automatically chosen by cluster config. This should be enabled in UI by default. Also it will be used for deserialization if no serdes passed." type: boolean schema: type: string @@ -2542,7 +2552,7 @@ components: partitions: type: array items: - $ref: '#/components/schemas/Partition' + $ref: "#/components/schemas/Partition" required: - name @@ -2586,7 +2596,7 @@ components: partitionStats: type: array items: - $ref: '#/components/schemas/TopicAnalysisStats' + $ref: "#/components/schemas/TopicAnalysisStats" TopicAnalysisStats: type: object @@ -2594,7 +2604,7 @@ components: partition: type: integer format: int32 - description: 'null if this is total stats' + description: "null if this is total stats" totalMsgs: type: integer format: int64 @@ -2623,9 +2633,9 @@ components: type: integer format: int64 keySize: - $ref: '#/components/schemas/TopicAnalysisSizeStats' + $ref: "#/components/schemas/TopicAnalysisSizeStats" valueSize: - $ref: '#/components/schemas/TopicAnalysisSizeStats' + $ref: "#/components/schemas/TopicAnalysisSizeStats" hourlyMsgCounts: type: array items: @@ -2640,7 +2650,7 @@ components: TopicAnalysisSizeStats: type: object - description: 'All sizes in bytes' + description: "All sizes in bytes" properties: sum: type: integer @@ -2690,7 +2700,7 @@ components: partitions: type: array items: - $ref: '#/components/schemas/Partition' + $ref: "#/components/schemas/Partition" partitionCount: type: integer replicationFactor: @@ -2729,7 +2739,7 @@ components: defaultValue: type: string source: - $ref: '#/components/schemas/ConfigSource' + $ref: "#/components/schemas/ConfigSource" isSensitive: type: boolean isReadOnly: @@ -2737,7 +2747,7 @@ components: synonyms: type: array items: - $ref: '#/components/schemas/ConfigSynonym' + $ref: "#/components/schemas/ConfigSynonym" doc: type: string required: @@ -2853,7 +2863,7 @@ components: discriminator: propertyName: inherit mapping: - details: '#/components/schemas/ConsumerGroupDetails' + details: "#/components/schemas/ConsumerGroupDetails" type: object properties: groupId: @@ -2867,9 +2877,9 @@ components: partitionAssignor: type: string state: - $ref: '#/components/schemas/ConsumerGroupState' + $ref: "#/components/schemas/ConsumerGroupState" coordinator: - $ref: '#/components/schemas/Broker' + $ref: "#/components/schemas/Broker" consumerLag: type: integer format: int64 @@ -2964,13 +2974,13 @@ components: # otherwise ui should pass cursor param to continue polling - DONE message: - $ref: '#/components/schemas/TopicMessage' + $ref: "#/components/schemas/TopicMessage" phase: - $ref: '#/components/schemas/TopicMessagePhase' + $ref: "#/components/schemas/TopicMessagePhase" consuming: - $ref: '#/components/schemas/TopicMessageConsuming' + $ref: "#/components/schemas/TopicMessageConsuming" cursor: - $ref: '#/components/schemas/TopicMessageNextPageCursor' + $ref: "#/components/schemas/TopicMessageNextPageCursor" TopicMessagePhase: type: object @@ -3033,10 +3043,10 @@ components: type: string keyFormat: #deprecated - wont be filled - use 'keySerde' field instead - $ref: '#/components/schemas/MessageFormat' + $ref: "#/components/schemas/MessageFormat" valueFormat: #deprecated - wont be filled - use 'valueSerde' field instead - $ref: '#/components/schemas/MessageFormat' + $ref: "#/components/schemas/MessageFormat" keySize: type: integer format: int64 @@ -3162,6 +3172,7 @@ components: - topic - partition + ConsumerGroupDetails: allOf: - $ref: '#/components/schemas/ConsumerGroup' @@ -3882,7 +3893,16 @@ components: KafkaAcl: type: object - required: [resourceType, resourceName, namePatternType, principal, host, operation, permission] + required: + [ + resourceType, + resourceName, + namePatternType, + principal, + host, + operation, + permission, + ] properties: resourceType: $ref: '#/components/schemas/KafkaAclResourceType' @@ -3899,15 +3919,15 @@ components: enum: - UNKNOWN # Unknown operation, need to update mapping code on BE - ALL # Cluster, Topic, Group - - READ # Topic, Group + - READ # Topic, Group - WRITE # Topic, TransactionalId - CREATE # Cluster, Topic - - DELETE # Topic, Group - - ALTER # Cluster, Topic, + - DELETE # Topic, Group + - ALTER # Cluster, Topic, - DESCRIBE # Cluster, Topic, Group, TransactionalId, DelegationToken - CLUSTER_ACTION # Cluster - DESCRIBE_CONFIGS # Cluster, Topic - - ALTER_CONFIGS # Cluster, Topic + - ALTER_CONFIGS # Cluster, Topic - IDEMPOTENT_WRITE # Cluster - CREATE_TOKENS - DESCRIBE_TOKENS @@ -4138,7 +4158,7 @@ components: properties: maxInMemoryBufferSize: type: string - description: 'examples: 20, 12KB, 5MB' + description: "examples: 20, 12KB, 5MB" kafka: type: object properties: @@ -4195,7 +4215,7 @@ components: keystoreLocation: type: string keystorePassword: - type: string + type: string ksqldbServerAuth: type: object properties: @@ -4303,7 +4323,7 @@ components: properties: level: type: string - enum: ['ALL', 'ALTER_ONLY'] + enum: [ "ALL", "ALTER_ONLY" ] topic: type: string auditTopicsPartitions: From 193daa06b219c27006df0059561f484d94b6e00c Mon Sep 17 00:00:00 2001 From: NOZAIS Julien Date: Mon, 7 Oct 2024 10:47:49 +0200 Subject: [PATCH 13/34] Revert formatting changes in "Add a dropdown option to reset the offsets of stopped connectors" This reverts commit 503a2a1efb13e2806f8392abdaa048c2813dee78. --- .../ui/controller/KafkaConnectController.java | 107 ++++++----- .../ui/service/KafkaConnectService.java | 168 ++++++++++-------- .../resources/swagger/kafka-connect-api.yaml | 2 + 3 files changed, 145 insertions(+), 132 deletions(-) diff --git a/api/src/main/java/io/kafbat/ui/controller/KafkaConnectController.java b/api/src/main/java/io/kafbat/ui/controller/KafkaConnectController.java index e992d4d46..44e609aed 100644 --- a/api/src/main/java/io/kafbat/ui/controller/KafkaConnectController.java +++ b/api/src/main/java/io/kafbat/ui/controller/KafkaConnectController.java @@ -34,15 +34,15 @@ @RequiredArgsConstructor @Slf4j public class KafkaConnectController extends AbstractController implements KafkaConnectApi { - private static final Set RESTART_ACTIONS = Set.of(RESTART, RESTART_FAILED_TASKS, - RESTART_ALL_TASKS); + private static final Set RESTART_ACTIONS + = Set.of(RESTART, RESTART_FAILED_TASKS, RESTART_ALL_TASKS); private static final String CONNECTOR_NAME = "connectorName"; private final KafkaConnectService kafkaConnectService; @Override public Mono>> getConnects(String clusterName, - ServerWebExchange exchange) { + ServerWebExchange exchange) { Flux availableConnects = kafkaConnectService.getConnects(getCluster(clusterName)) .filterWhen(dto -> accessControlService.isConnectAccessible(dto, clusterName)); @@ -52,7 +52,7 @@ public Mono>> getConnects(String clusterName, @Override public Mono>> getConnectors(String clusterName, String connectName, - ServerWebExchange exchange) { + ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) @@ -61,16 +61,14 @@ public Mono>> getConnectors(String clusterName, Stri .build(); return validateAccess(context) - .thenReturn( - ResponseEntity.ok(kafkaConnectService.getConnectorNames( - getCluster(clusterName), connectName))) + .thenReturn(ResponseEntity.ok(kafkaConnectService.getConnectorNames(getCluster(clusterName), connectName))) .doOnEach(sig -> audit(context, sig)); } @Override public Mono> createConnector(String clusterName, String connectName, - @Valid Mono connector, - ServerWebExchange exchange) { + @Valid Mono connector, + ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) @@ -80,14 +78,14 @@ public Mono> createConnector(String clusterName, St return validateAccess(context).then( kafkaConnectService.createConnector(getCluster(clusterName), connectName, connector) - .map(ResponseEntity::ok)) - .doOnEach(sig -> audit(context, sig)); + .map(ResponseEntity::ok) + ).doOnEach(sig -> audit(context, sig)); } @Override public Mono> getConnector(String clusterName, String connectName, - String connectorName, - ServerWebExchange exchange) { + String connectorName, + ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) @@ -97,14 +95,14 @@ public Mono> getConnector(String clusterName, Strin return validateAccess(context).then( kafkaConnectService.getConnector(getCluster(clusterName), connectName, connectorName) - .map(ResponseEntity::ok)) - .doOnEach(sig -> audit(context, sig)); + .map(ResponseEntity::ok) + ).doOnEach(sig -> audit(context, sig)); } @Override public Mono> deleteConnector(String clusterName, String connectName, - String connectorName, - ServerWebExchange exchange) { + String connectorName, + ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) @@ -115,17 +113,19 @@ public Mono> deleteConnector(String clusterName, String con return validateAccess(context).then( kafkaConnectService.deleteConnector(getCluster(clusterName), connectName, connectorName) - .map(ResponseEntity::ok)) - .doOnEach(sig -> audit(context, sig)); + .map(ResponseEntity::ok) + ).doOnEach(sig -> audit(context, sig)); } + @Override public Mono>> getAllConnectors( String clusterName, String search, ConnectorColumnsToSortDTO orderBy, SortOrderDTO sortOrder, - ServerWebExchange exchange) { + ServerWebExchange exchange + ) { var context = AccessContext.builder() .cluster(clusterName) .operationName("getAllConnectors") @@ -146,9 +146,9 @@ public Mono>> getAllConnectors( @Override public Mono>> getConnectorConfig(String clusterName, - String connectName, - String connectorName, - ServerWebExchange exchange) { + String connectName, + String connectorName, + ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) @@ -159,15 +159,15 @@ public Mono>> getConnectorConfig(String clust return validateAccess(context).then( kafkaConnectService .getConnectorConfig(getCluster(clusterName), connectName, connectorName) - .map(ResponseEntity::ok)) - .doOnEach(sig -> audit(context, sig)); + .map(ResponseEntity::ok) + ).doOnEach(sig -> audit(context, sig)); } @Override public Mono> setConnectorConfig(String clusterName, String connectName, - String connectorName, - Mono> requestBody, - ServerWebExchange exchange) { + String connectorName, + Mono> requestBody, + ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) @@ -177,23 +177,22 @@ public Mono> setConnectorConfig(String clusterName, .build(); return validateAccess(context).then( - kafkaConnectService - .setConnectorConfig(getCluster(clusterName), connectName, connectorName, - requestBody) - .map(ResponseEntity::ok)) + kafkaConnectService + .setConnectorConfig(getCluster(clusterName), connectName, connectorName, requestBody) + .map(ResponseEntity::ok)) .doOnEach(sig -> audit(context, sig)); } @Override public Mono> updateConnectorState(String clusterName, String connectName, - String connectorName, - ConnectorActionDTO action, - ServerWebExchange exchange) { + String connectorName, + ConnectorActionDTO action, + ServerWebExchange exchange) { ConnectAction[] connectActions; if (RESTART_ACTIONS.contains(action)) { - connectActions = new ConnectAction[] { ConnectAction.VIEW, ConnectAction.RESTART }; + connectActions = new ConnectAction[] {ConnectAction.VIEW, ConnectAction.RESTART}; } else { - connectActions = new ConnectAction[] { ConnectAction.VIEW, ConnectAction.EDIT }; + connectActions = new ConnectAction[] {ConnectAction.VIEW, ConnectAction.EDIT}; } var context = AccessContext.builder() @@ -205,17 +204,16 @@ public Mono> updateConnectorState(String clusterName, Strin return validateAccess(context).then( kafkaConnectService - .updateConnectorState(getCluster(clusterName), connectName, - connectorName, action) - .map(ResponseEntity::ok)) - .doOnEach(sig -> audit(context, sig)); + .updateConnectorState(getCluster(clusterName), connectName, connectorName, action) + .map(ResponseEntity::ok) + ).doOnEach(sig -> audit(context, sig)); } @Override public Mono>> getConnectorTasks(String clusterName, - String connectName, - String connectorName, - ServerWebExchange exchange) { + String connectName, + String connectorName, + ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) .connectActions(connectName, ConnectAction.VIEW) @@ -226,15 +224,14 @@ public Mono>> getConnectorTasks(String clusterName, return validateAccess(context).thenReturn( ResponseEntity .ok(kafkaConnectService - .getConnectorTasks(getCluster(clusterName), connectName, - connectorName))) - .doOnEach(sig -> audit(context, sig)); + .getConnectorTasks(getCluster(clusterName), connectName, connectorName)) + ).doOnEach(sig -> audit(context, sig)); } @Override public Mono> restartConnectorTask(String clusterName, String connectName, - String connectorName, Integer taskId, - ServerWebExchange exchange) { + String connectorName, Integer taskId, + ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) @@ -245,10 +242,9 @@ public Mono> restartConnectorTask(String clusterName, Strin return validateAccess(context).then( kafkaConnectService - .restartConnectorTask(getCluster(clusterName), connectName, - connectorName, taskId) - .map(ResponseEntity::ok)) - .doOnEach(sig -> audit(context, sig)); + .restartConnectorTask(getCluster(clusterName), connectName, connectorName, taskId) + .map(ResponseEntity::ok) + ).doOnEach(sig -> audit(context, sig)); } @Override @@ -264,9 +260,8 @@ public Mono>> getConnectorPlugins( return validateAccess(context).then( Mono.just( ResponseEntity.ok( - kafkaConnectService.getConnectorPlugins( - getCluster(clusterName), connectName)))) - .doOnEach(sig -> audit(context, sig)); + kafkaConnectService.getConnectorPlugins(getCluster(clusterName), connectName))) + ).doOnEach(sig -> audit(context, sig)); } @Override diff --git a/api/src/main/java/io/kafbat/ui/service/KafkaConnectService.java b/api/src/main/java/io/kafbat/ui/service/KafkaConnectService.java index 41b437616..8b74416d3 100644 --- a/api/src/main/java/io/kafbat/ui/service/KafkaConnectService.java +++ b/api/src/main/java/io/kafbat/ui/service/KafkaConnectService.java @@ -52,24 +52,28 @@ public Flux getConnects(KafkaCluster cluster) { return Flux.fromIterable( Optional.ofNullable(cluster.getOriginalProperties().getKafkaConnect()) .map(lst -> lst.stream().map(clusterMapper::toKafkaConnect).toList()) - .orElse(List.of())); + .orElse(List.of()) + ); } public Flux getAllConnectors(final KafkaCluster cluster, - @Nullable final String search) { + @Nullable final String search) { return getConnects(cluster) - .flatMap(connect -> getConnectorNamesWithErrorsSuppress(cluster, connect.getName()) - .flatMap(connectorName -> Mono.zip( - getConnector(cluster, connect.getName(), connectorName), - getConnectorConfig(cluster, connect.getName(), connectorName), - getConnectorTasks(cluster, connect.getName(), connectorName).collectList(), - getConnectorTopics(cluster, connect.getName(), connectorName)) - .map(tuple -> InternalConnectInfo.builder() - .connector(tuple.getT1()) - .config(tuple.getT2()) - .tasks(tuple.getT3()) - .topics(tuple.getT4().getTopics()) - .build()))) + .flatMap(connect -> + getConnectorNamesWithErrorsSuppress(cluster, connect.getName()) + .flatMap(connectorName -> + Mono.zip( + getConnector(cluster, connect.getName(), connectorName), + getConnectorConfig(cluster, connect.getName(), connectorName), + getConnectorTasks(cluster, connect.getName(), connectorName).collectList(), + getConnectorTopics(cluster, connect.getName(), connectorName) + ).map(tuple -> + InternalConnectInfo.builder() + .connector(tuple.getT1()) + .config(tuple.getT2()) + .tasks(tuple.getT3()) + .topics(tuple.getT4().getTopics()) + .build()))) .map(kafkaConnectMapper::fullConnectorInfo) .filter(matchesSearchTerm(search)); } @@ -91,7 +95,7 @@ private Stream getStringsForSearch(FullConnectorInfoDTO fullConnectorInf } public Mono getConnectorTopics(KafkaCluster cluster, String connectClusterName, - String connectorName) { + String connectorName) { return api(cluster, connectClusterName) .mono(c -> c.getConnectorTopics(connectorName)) .map(result -> result.get(connectorName)) @@ -103,8 +107,7 @@ public Mono getConnectorTopics(KafkaCluster cluster, String con public Flux getConnectorNames(KafkaCluster cluster, String connectName) { return api(cluster, connectName) .flux(client -> client.getConnectors(null)) - // for some reason `getConnectors` method returns the response as a single - // string + // for some reason `getConnectors` method returns the response as a single string .collectList().map(e -> e.get(0)) .map(this::parseConnectorsNamesStringToList) .flatMapMany(Flux::fromIterable); @@ -122,59 +125,64 @@ private List parseConnectorsNamesStringToList(String json) { } public Mono createConnector(KafkaCluster cluster, String connectName, - Mono connector) { + Mono connector) { return api(cluster, connectName) - .mono(client -> connector - .flatMap(c -> connectorExists(cluster, connectName, c.getName()) - .map(exists -> { - if (Boolean.TRUE.equals(exists)) { - throw new ValidationException( - String.format("Connector with name %s already exists", c.getName())); - } - return c; - })) - .map(kafkaConnectMapper::toClient) - .flatMap(client::createConnector) - .flatMap(c -> getConnector(cluster, connectName, c.getName()))); + .mono(client -> + connector + .flatMap(c -> connectorExists(cluster, connectName, c.getName()) + .map(exists -> { + if (Boolean.TRUE.equals(exists)) { + throw new ValidationException( + String.format("Connector with name %s already exists", c.getName())); + } + return c; + })) + .map(kafkaConnectMapper::toClient) + .flatMap(client::createConnector) + .flatMap(c -> getConnector(cluster, connectName, c.getName())) + ); } private Mono connectorExists(KafkaCluster cluster, String connectName, - String connectorName) { + String connectorName) { return getConnectorNames(cluster, connectName) .any(name -> name.equals(connectorName)); } public Mono getConnector(KafkaCluster cluster, String connectName, - String connectorName) { + String connectorName) { return api(cluster, connectName) .mono(client -> client.getConnector(connectorName) .map(kafkaConnectMapper::fromClient) - .flatMap(connector -> client.getConnectorStatus(connector.getName()) - // status request can return 404 if tasks not assigned yet - .onErrorResume(WebClientResponseException.NotFound.class, - e -> emptyStatus(connectorName)) - .map(connectorStatus -> { - var status = connectorStatus.getConnector(); - var sanitizedConfig = kafkaConfigSanitizer.sanitizeConnectorConfig(connector.getConfig()); - ConnectorDTO result = new ConnectorDTO() - .connect(connectName) - .status(kafkaConnectMapper.fromClient(status)) - .type(connector.getType()) - .tasks(connector.getTasks()) - .name(connector.getName()) - .config(sanitizedConfig); + .flatMap(connector -> + client.getConnectorStatus(connector.getName()) + // status request can return 404 if tasks not assigned yet + .onErrorResume(WebClientResponseException.NotFound.class, + e -> emptyStatus(connectorName)) + .map(connectorStatus -> { + var status = connectorStatus.getConnector(); + var sanitizedConfig = kafkaConfigSanitizer.sanitizeConnectorConfig(connector.getConfig()); + ConnectorDTO result = new ConnectorDTO() + .connect(connectName) + .status(kafkaConnectMapper.fromClient(status)) + .type(connector.getType()) + .tasks(connector.getTasks()) + .name(connector.getName()) + .config(sanitizedConfig); - if (connectorStatus.getTasks() != null) { - boolean isAnyTaskFailed = connectorStatus.getTasks().stream() - .map(TaskStatus::getState) - .anyMatch(TaskStatus.StateEnum.FAILED::equals); + if (connectorStatus.getTasks() != null) { + boolean isAnyTaskFailed = connectorStatus.getTasks().stream() + .map(TaskStatus::getState) + .anyMatch(TaskStatus.StateEnum.FAILED::equals); - if (isAnyTaskFailed) { - result.getStatus().state(ConnectorStateDTO.TASK_FAILED); - } - } - return result; - }))); + if (isAnyTaskFailed) { + result.getStatus().state(ConnectorStateDTO.TASK_FAILED); + } + } + return result; + }) + ) + ); } private Mono emptyStatus(String connectorName) { @@ -186,18 +194,19 @@ private Mono emptyStatus(String connectorName) { } public Mono> getConnectorConfig(KafkaCluster cluster, String connectName, - String connectorName) { + String connectorName) { return api(cluster, connectName) .mono(c -> c.getConnectorConfig(connectorName)) .map(kafkaConfigSanitizer::sanitizeConnectorConfig); } public Mono setConnectorConfig(KafkaCluster cluster, String connectName, - String connectorName, Mono> requestBody) { + String connectorName, Mono> requestBody) { return api(cluster, connectName) - .mono(c -> requestBody - .flatMap(body -> c.setConnectorConfig(connectorName, body)) - .map(kafkaConnectMapper::fromClient)); + .mono(c -> + requestBody + .flatMap(body -> c.setConnectorConfig(connectorName, body)) + .map(kafkaConnectMapper::fromClient)); } public Mono deleteConnector( @@ -207,7 +216,7 @@ public Mono deleteConnector( } public Mono updateConnectorState(KafkaCluster cluster, String connectName, - String connectorName, ConnectorActionDTO action) { + String connectorName, ConnectorActionDTO action) { return api(cluster, connectName) .mono(client -> { switch (action) { @@ -231,33 +240,37 @@ public Mono updateConnectorState(KafkaCluster cluster, String connectName, } private Mono restartTasks(KafkaCluster cluster, String connectName, - String connectorName, Predicate taskFilter) { + String connectorName, Predicate taskFilter) { return getConnectorTasks(cluster, connectName, connectorName) .filter(taskFilter) - .flatMap(t -> restartConnectorTask(cluster, connectName, connectorName, t.getId().getTask())) + .flatMap(t -> + restartConnectorTask(cluster, connectName, connectorName, t.getId().getTask())) .then(); } public Flux getConnectorTasks(KafkaCluster cluster, String connectName, String connectorName) { return api(cluster, connectName) - .flux(client -> client.getConnectorTasks(connectorName) - .onErrorResume(WebClientResponseException.NotFound.class, e -> Flux.empty()) - .map(kafkaConnectMapper::fromClient) - .flatMap(task -> client - .getConnectorTaskStatus(connectorName, task.getId().getTask()) - .onErrorResume(WebClientResponseException.NotFound.class, e -> Mono.empty()) + .flux(client -> + client.getConnectorTasks(connectorName) + .onErrorResume(WebClientResponseException.NotFound.class, e -> Flux.empty()) .map(kafkaConnectMapper::fromClient) - .map(task::status))); + .flatMap(task -> + client + .getConnectorTaskStatus(connectorName, task.getId().getTask()) + .onErrorResume(WebClientResponseException.NotFound.class, e -> Mono.empty()) + .map(kafkaConnectMapper::fromClient) + .map(task::status) + )); } public Mono restartConnectorTask(KafkaCluster cluster, String connectName, - String connectorName, Integer taskId) { + String connectorName, Integer taskId) { return api(cluster, connectName) .mono(client -> client.restartConnectorTask(connectorName, taskId)); } public Flux getConnectorPlugins(KafkaCluster cluster, - String connectName) { + String connectName) { return api(cluster, connectName) .flux(client -> client.getConnectorPlugins().map(kafkaConnectMapper::fromClient)); } @@ -265,9 +278,12 @@ public Flux getConnectorPlugins(KafkaCluster cluster, public Mono validateConnectorPluginConfig( KafkaCluster cluster, String connectName, String pluginName, Mono> requestBody) { return api(cluster, connectName) - .mono(client -> requestBody - .flatMap(body -> client.validateConnectorPluginConfig(pluginName, body)) - .map(kafkaConnectMapper::fromClient)); + .mono(client -> + requestBody + .flatMap(body -> + client.validateConnectorPluginConfig(pluginName, body)) + .map(kafkaConnectMapper::fromClient) + ); } private ReactiveFailover api(KafkaCluster cluster, String connectName) { diff --git a/contract/src/main/resources/swagger/kafka-connect-api.yaml b/contract/src/main/resources/swagger/kafka-connect-api.yaml index 39e34a0d6..cfe81bff9 100644 --- a/contract/src/main/resources/swagger/kafka-connect-api.yaml +++ b/contract/src/main/resources/swagger/kafka-connect-api.yaml @@ -606,5 +606,7 @@ components: items: type: string + security: - basicAuth: [] + From 05df53e94cc79bfd41758d637a0d3b4b11f99ab4 Mon Sep 17 00:00:00 2001 From: NOZAIS Julien Date: Mon, 7 Oct 2024 10:52:34 +0200 Subject: [PATCH 14/34] Revert formatting changes in "Fix code formatting and typos" This reverts commit c9d8aad931408d1fd27c4063b29a08a303edea19. --- .../ui/client/RetryingKafkaConnectClient.java | 45 +++++++++++-------- .../ui/controller/KafkaConnectController.java | 37 +++++++-------- .../Actions/__tests__/Actions.spec.tsx | 4 +- .../components/Connect/List/ActionsCell.tsx | 3 +- 4 files changed, 45 insertions(+), 44 deletions(-) diff --git a/api/src/main/java/io/kafbat/ui/client/RetryingKafkaConnectClient.java b/api/src/main/java/io/kafbat/ui/client/RetryingKafkaConnectClient.java index eeac1f1e9..2a13cb961 100644 --- a/api/src/main/java/io/kafbat/ui/client/RetryingKafkaConnectClient.java +++ b/api/src/main/java/io/kafbat/ui/client/RetryingKafkaConnectClient.java @@ -34,8 +34,8 @@ public class RetryingKafkaConnectClient extends KafkaConnectClientApi { private static final Duration RETRIES_DELAY = Duration.ofMillis(200); public RetryingKafkaConnectClient(ClustersProperties.ConnectCluster config, - @Nullable ClustersProperties.TruststoreConfig truststoreConfig, - DataSize maxBuffSize) { + @Nullable ClustersProperties.TruststoreConfig truststoreConfig, + DataSize maxBuffSize) { super(new RetryingApiClient(config, truststoreConfig, maxBuffSize)); } @@ -43,8 +43,9 @@ private static Retry conflictCodeRetry() { return Retry .fixedDelay(MAX_RETRIES, RETRIES_DELAY) .filter(e -> e instanceof WebClientResponseException.Conflict) - .onRetryExhaustedThrow((spec, signal) -> new KafkaConnectConflictReponseException( - (WebClientResponseException.Conflict) signal.failure())); + .onRetryExhaustedThrow((spec, signal) -> + new KafkaConnectConflictReponseException( + (WebClientResponseException.Conflict) signal.failure())); } private static Mono withRetryOnConflict(Mono publisher) { @@ -57,23 +58,25 @@ private static Flux withRetryOnConflict(Flux publisher) { private static Mono withBadRequestErrorHandling(Mono publisher) { return publisher - .onErrorResume(WebClientResponseException.BadRequest.class, - e -> Mono.error(new ValidationException("Invalid configuration"))) - .onErrorResume(WebClientResponseException.InternalServerError.class, - e -> Mono.error(new ValidationException("Invalid configuration"))); + .onErrorResume(WebClientResponseException.BadRequest.class, e -> + Mono.error(new ValidationException("Invalid configuration"))) + .onErrorResume(WebClientResponseException.InternalServerError.class, e -> + Mono.error(new ValidationException("Invalid configuration"))); } @Override public Mono createConnector(NewConnector newConnector) throws RestClientException { return withBadRequestErrorHandling( - super.createConnector(newConnector)); + super.createConnector(newConnector) + ); } @Override public Mono setConnectorConfig(String connectorName, Map requestBody) throws RestClientException { return withBadRequestErrorHandling( - super.setConnectorConfig(connectorName, requestBody)); + super.setConnectorConfig(connectorName, requestBody) + ); } @Override @@ -93,6 +96,7 @@ public Mono> deleteConnectorWithHttpInfo(String connectorNa return withRetryOnConflict(super.deleteConnectorWithHttpInfo(connectorName)); } + @Override public Mono getConnector(String connectorName) throws WebClientResponseException { return withRetryOnConflict(super.getConnector(connectorName)); @@ -204,7 +208,7 @@ public Mono restartConnector(String connectorName, Boolean includeTasks, B @Override public Mono> restartConnectorWithHttpInfo(String connectorName, Boolean includeTasks, - Boolean onlyFailed) throws WebClientResponseException { + Boolean onlyFailed) throws WebClientResponseException { return withRetryOnConflict(super.restartConnectorWithHttpInfo(connectorName, includeTasks, onlyFailed)); } @@ -232,14 +236,14 @@ public Mono> resumeConnectorWithHttpInfo(String connectorNa @Override public Mono> setConnectorConfigWithHttpInfo(String connectorName, - Map requestBody) + Map requestBody) throws WebClientResponseException { return withRetryOnConflict(super.setConnectorConfigWithHttpInfo(connectorName, requestBody)); } @Override public Mono validateConnectorPluginConfig(String pluginName, - Map requestBody) + Map requestBody) throws WebClientResponseException { return withRetryOnConflict(super.validateConnectorPluginConfig(pluginName, requestBody)); } @@ -253,8 +257,8 @@ public Mono> validateCon private static class RetryingApiClient extends ApiClient { public RetryingApiClient(ClustersProperties.ConnectCluster config, - ClustersProperties.TruststoreConfig truststoreConfig, - DataSize maxBuffSize) { + ClustersProperties.TruststoreConfig truststoreConfig, + DataSize maxBuffSize) { super(buildWebClient(maxBuffSize, config, truststoreConfig), null, null); setBasePath(config.getAddress()); setUsername(config.getUsername()); @@ -262,17 +266,20 @@ public RetryingApiClient(ClustersProperties.ConnectCluster config, } public static WebClient buildWebClient(DataSize maxBuffSize, - ClustersProperties.ConnectCluster config, - ClustersProperties.TruststoreConfig truststoreConfig) { + ClustersProperties.ConnectCluster config, + ClustersProperties.TruststoreConfig truststoreConfig) { return new WebClientConfigurator() .configureSsl( truststoreConfig, new ClustersProperties.KeystoreConfig( config.getKeystoreLocation(), - config.getKeystorePassword())) + config.getKeystorePassword() + ) + ) .configureBasicAuth( config.getUsername(), - config.getPassword()) + config.getPassword() + ) .configureBufferSize(maxBuffSize) .build(); } diff --git a/api/src/main/java/io/kafbat/ui/controller/KafkaConnectController.java b/api/src/main/java/io/kafbat/ui/controller/KafkaConnectController.java index 44e609aed..8d9fdc8ff 100644 --- a/api/src/main/java/io/kafbat/ui/controller/KafkaConnectController.java +++ b/api/src/main/java/io/kafbat/ui/controller/KafkaConnectController.java @@ -61,7 +61,8 @@ public Mono>> getConnectors(String clusterName, Stri .build(); return validateAccess(context) - .thenReturn(ResponseEntity.ok(kafkaConnectService.getConnectorNames(getCluster(clusterName), connectName))) + .thenReturn( + ResponseEntity.ok(kafkaConnectService.getConnectorNames(getCluster(clusterName), connectName))) .doOnEach(sig -> audit(context, sig)); } @@ -136,8 +137,7 @@ public Mono>> getAllConnectors( : getConnectorsComparator(orderBy).reversed(); Flux job = kafkaConnectService.getAllConnectors(getCluster(clusterName), search) - .filterWhen(dto -> accessControlService.isConnectAccessible(dto.getConnect(), - clusterName)) + .filterWhen(dto -> accessControlService.isConnectAccessible(dto.getConnect(), clusterName)) .sort(comparator); return Mono.just(ResponseEntity.ok(job)) @@ -177,9 +177,9 @@ public Mono> setConnectorConfig(String clusterName, .build(); return validateAccess(context).then( - kafkaConnectService - .setConnectorConfig(getCluster(clusterName), connectName, connectorName, requestBody) - .map(ResponseEntity::ok)) + kafkaConnectService + .setConnectorConfig(getCluster(clusterName), connectName, connectorName, requestBody) + .map(ResponseEntity::ok)) .doOnEach(sig -> audit(context, sig)); } @@ -205,8 +205,8 @@ public Mono> updateConnectorState(String clusterName, Strin return validateAccess(context).then( kafkaConnectService .updateConnectorState(getCluster(clusterName), connectName, connectorName, action) - .map(ResponseEntity::ok) - ).doOnEach(sig -> audit(context, sig)); + .map(ResponseEntity::ok)) + .doOnEach(sig -> audit(context, sig)); } @Override @@ -224,8 +224,8 @@ public Mono>> getConnectorTasks(String clusterName, return validateAccess(context).thenReturn( ResponseEntity .ok(kafkaConnectService - .getConnectorTasks(getCluster(clusterName), connectName, connectorName)) - ).doOnEach(sig -> audit(context, sig)); + .getConnectorTasks(getCluster(clusterName), connectName, connectorName))) + .doOnEach(sig -> audit(context, sig)); } @Override @@ -243,8 +243,8 @@ public Mono> restartConnectorTask(String clusterName, Strin return validateAccess(context).then( kafkaConnectService .restartConnectorTask(getCluster(clusterName), connectName, connectorName, taskId) - .map(ResponseEntity::ok) - ).doOnEach(sig -> audit(context, sig)); + .map(ResponseEntity::ok)) + .doOnEach(sig -> audit(context, sig)); } @Override @@ -260,14 +260,13 @@ public Mono>> getConnectorPlugins( return validateAccess(context).then( Mono.just( ResponseEntity.ok( - kafkaConnectService.getConnectorPlugins(getCluster(clusterName), connectName))) - ).doOnEach(sig -> audit(context, sig)); + kafkaConnectService.getConnectorPlugins(getCluster(clusterName), connectName)))) + .doOnEach(sig -> audit(context, sig)); } @Override public Mono> validateConnectorPluginConfig( - String clusterName, String connectName, String pluginName, - @Valid Mono> requestBody, + String clusterName, String connectName, String pluginName, @Valid Mono> requestBody, ServerWebExchange exchange) { return kafkaConnectService .validateConnectorPluginConfig( @@ -283,8 +282,7 @@ private Comparator getConnectorsComparator(ConnectorColumn return switch (orderBy) { case CONNECT -> Comparator.comparing(FullConnectorInfoDTO::getConnect); case TYPE -> Comparator.comparing(FullConnectorInfoDTO::getType); - case STATUS -> Comparator - .comparing(fullConnectorInfoDTO -> fullConnectorInfoDTO.getStatus().getState()); + case STATUS -> Comparator.comparing(fullConnectorInfoDTO -> fullConnectorInfoDTO.getStatus().getState()); default -> defaultComparator; }; } @@ -306,8 +304,7 @@ public Mono> resetConnectorOffsets(String clusterName, Stri return validateAccess(context).then( kafkaConnectService - .resetConnectorOffsets(getCluster(clusterName), connectName, - connectorName) + .resetConnectorOffsets(getCluster(clusterName), connectName, connectorName) .map(ResponseEntity::ok)) .doOnEach(sig -> audit(context, sig)); } diff --git a/frontend/src/components/Connect/Details/Actions/__tests__/Actions.spec.tsx b/frontend/src/components/Connect/Details/Actions/__tests__/Actions.spec.tsx index 34c574008..0f5582992 100644 --- a/frontend/src/components/Connect/Details/Actions/__tests__/Actions.spec.tsx +++ b/frontend/src/components/Connect/Details/Actions/__tests__/Actions.spec.tsx @@ -172,9 +172,7 @@ describe('Actions', () => { renderComponent(); await afterClickDropDownButton(); await waitFor(async () => - userEvent.click( - screen.getByRole('menuitem', { name: 'Reset Offsets' }) - ) + userEvent.click(screen.getByRole('menuitem', { name: 'Reset Offsets' })) ); expect(screen.getByRole('dialog')).toBeInTheDocument(); }); diff --git a/frontend/src/components/Connect/List/ActionsCell.tsx b/frontend/src/components/Connect/List/ActionsCell.tsx index 4ad50c00e..720e9be47 100644 --- a/frontend/src/components/Connect/List/ActionsCell.tsx +++ b/frontend/src/components/Connect/List/ActionsCell.tsx @@ -173,8 +173,7 @@ const ActionsCell: React.FC> = ({ resource: ResourceType.CONNECT, action: Action.DELETE, value: connect, - }} - > + }}> Delete From 7b289c31f0eb241faafb0f9f91c5640d4daa8c84 Mon Sep 17 00:00:00 2001 From: NOZAIS Julien Date: Mon, 7 Oct 2024 11:00:40 +0200 Subject: [PATCH 15/34] revert formatting changes to the controller --- .../ui/controller/KafkaConnectController.java | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/api/src/main/java/io/kafbat/ui/controller/KafkaConnectController.java b/api/src/main/java/io/kafbat/ui/controller/KafkaConnectController.java index 8d9fdc8ff..209239a7c 100644 --- a/api/src/main/java/io/kafbat/ui/controller/KafkaConnectController.java +++ b/api/src/main/java/io/kafbat/ui/controller/KafkaConnectController.java @@ -61,8 +61,7 @@ public Mono>> getConnectors(String clusterName, Stri .build(); return validateAccess(context) - .thenReturn( - ResponseEntity.ok(kafkaConnectService.getConnectorNames(getCluster(clusterName), connectName))) + .thenReturn(ResponseEntity.ok(kafkaConnectService.getConnectorNames(getCluster(clusterName), connectName))) .doOnEach(sig -> audit(context, sig)); } @@ -177,9 +176,9 @@ public Mono> setConnectorConfig(String clusterName, .build(); return validateAccess(context).then( - kafkaConnectService - .setConnectorConfig(getCluster(clusterName), connectName, connectorName, requestBody) - .map(ResponseEntity::ok)) + kafkaConnectService + .setConnectorConfig(getCluster(clusterName), connectName, connectorName, requestBody) + .map(ResponseEntity::ok)) .doOnEach(sig -> audit(context, sig)); } @@ -205,8 +204,8 @@ public Mono> updateConnectorState(String clusterName, Strin return validateAccess(context).then( kafkaConnectService .updateConnectorState(getCluster(clusterName), connectName, connectorName, action) - .map(ResponseEntity::ok)) - .doOnEach(sig -> audit(context, sig)); + .map(ResponseEntity::ok) + ).doOnEach(sig -> audit(context, sig)); } @Override @@ -224,8 +223,8 @@ public Mono>> getConnectorTasks(String clusterName, return validateAccess(context).thenReturn( ResponseEntity .ok(kafkaConnectService - .getConnectorTasks(getCluster(clusterName), connectName, connectorName))) - .doOnEach(sig -> audit(context, sig)); + .getConnectorTasks(getCluster(clusterName), connectName, connectorName)) + ).doOnEach(sig -> audit(context, sig)); } @Override @@ -243,8 +242,8 @@ public Mono> restartConnectorTask(String clusterName, Strin return validateAccess(context).then( kafkaConnectService .restartConnectorTask(getCluster(clusterName), connectName, connectorName, taskId) - .map(ResponseEntity::ok)) - .doOnEach(sig -> audit(context, sig)); + .map(ResponseEntity::ok) + ).doOnEach(sig -> audit(context, sig)); } @Override @@ -260,8 +259,8 @@ public Mono>> getConnectorPlugins( return validateAccess(context).then( Mono.just( ResponseEntity.ok( - kafkaConnectService.getConnectorPlugins(getCluster(clusterName), connectName)))) - .doOnEach(sig -> audit(context, sig)); + kafkaConnectService.getConnectorPlugins(getCluster(clusterName), connectName))) + ).doOnEach(sig -> audit(context, sig)); } @Override From 0d695cd9410677c2263f842b23d3e010730a1163 Mon Sep 17 00:00:00 2001 From: NOZAIS Julien Date: Mon, 7 Oct 2024 13:55:14 +0200 Subject: [PATCH 16/34] fix frontend tests --- .../Actions/__tests__/Actions.spec.tsx | 34 +++++++++------- .../Connect/List/__tests__/List.spec.tsx | 39 ++++++++++++++++++- frontend/src/lib/fixtures/kafkaConnect.ts | 12 ++++++ 3 files changed, 71 insertions(+), 14 deletions(-) diff --git a/frontend/src/components/Connect/Details/Actions/__tests__/Actions.spec.tsx b/frontend/src/components/Connect/Details/Actions/__tests__/Actions.spec.tsx index 0f5582992..e2588e8d9 100644 --- a/frontend/src/components/Connect/Details/Actions/__tests__/Actions.spec.tsx +++ b/frontend/src/components/Connect/Details/Actions/__tests__/Actions.spec.tsx @@ -87,14 +87,15 @@ describe('Actions', () => { expect(screen.getByText('Resume')).toBeInTheDocument(); expect(screen.queryByText('Pause')).not.toBeInTheDocument(); expect(screen.queryByText('Stop')).not.toBeInTheDocument(); - expect(screen.queryByText('Reset Offsets')).toBeInTheDocument(); - expect(screen.getByText('Reset Offsets')).toBeDisabled(); + await afterClickDropDownButton(); + expect(screen.getByText('Reset Offsets')).toBeInTheDocument(); + expect(screen.getByRole('menuitem', { name: 'Reset Offsets' })).toHaveAttribute('aria-disabled'); expectActionButtonsExists(); }); it('renders buttons when stopped', async () => { (useConnector as jest.Mock).mockImplementation(() => ({ - data: setConnectorStatus(connector, ConnectorState.PAUSED), + data: setConnectorStatus(connector, ConnectorState.STOPPED), })); renderComponent(); await afterClickRestartButton(); @@ -102,8 +103,9 @@ describe('Actions', () => { expect(screen.getByText('Resume')).toBeInTheDocument(); expect(screen.queryByText('Pause')).not.toBeInTheDocument(); expect(screen.queryByText('Stop')).not.toBeInTheDocument(); - expect(screen.queryByText('Reset Offsets')).toBeInTheDocument(); - expect(screen.getByText('Reset Offsets')).toBeEnabled(); + await afterClickDropDownButton(); + expect(screen.getByText('Reset Offsets')).toBeInTheDocument(); + expect(screen.getByRole('menuitem', { name: 'Reset Offsets' })).not.toHaveAttribute('aria-disabled'); expectActionButtonsExists(); }); @@ -117,8 +119,9 @@ describe('Actions', () => { expect(screen.queryByText('Resume')).not.toBeInTheDocument(); expect(screen.queryByText('Pause')).not.toBeInTheDocument(); expect(screen.queryByText('Stop')).not.toBeInTheDocument(); - expect(screen.queryByText('Reset Offsets')).toBeInTheDocument(); - expect(screen.getByText('Reset Offsets')).toBeDisabled(); + await afterClickDropDownButton(); + expect(screen.getByText('Reset Offsets')).toBeInTheDocument(); + expect(screen.getByRole('menuitem', { name: 'Reset Offsets' })).toHaveAttribute('aria-disabled'); expectActionButtonsExists(); }); @@ -132,8 +135,9 @@ describe('Actions', () => { expect(screen.queryByText('Resume')).not.toBeInTheDocument(); expect(screen.queryByText('Pause')).not.toBeInTheDocument(); expect(screen.queryByText('Stop')).not.toBeInTheDocument(); - expect(screen.queryByText('Reset Offsets')).toBeInTheDocument(); - expect(screen.getByText('Reset Offsets')).toBeDisabled(); + await afterClickDropDownButton(); + expect(screen.getByText('Reset Offsets')).toBeInTheDocument(); + expect(screen.getByRole('menuitem', { name: 'Reset Offsets' })).toHaveAttribute('aria-disabled'); expectActionButtonsExists(); }); @@ -143,12 +147,13 @@ describe('Actions', () => { })); renderComponent(); await afterClickRestartButton(); - expect(screen.getAllByRole('menuitem').length).toEqual(4); + expect(screen.getAllByRole('menuitem').length).toEqual(5); expect(screen.queryByText('Resume')).not.toBeInTheDocument(); expect(screen.getByText('Pause')).toBeInTheDocument(); expect(screen.getByText('Stop')).toBeInTheDocument(); - expect(screen.queryByText('Reset Offsets')).toBeInTheDocument(); - expect(screen.getByText('Reset Offsets')).toBeDisabled(); + await afterClickDropDownButton(); + expect(screen.getByText('Reset Offsets')).toBeInTheDocument(); + expect(screen.getByRole('menuitem', { name: 'Reset Offsets' })).toHaveAttribute('aria-disabled'); expectActionButtonsExists(); }); @@ -168,7 +173,10 @@ describe('Actions', () => { expect(screen.getByRole('dialog')).toBeInTheDocument(); }); - it('opens confirmation modal when reset offsets button clicked', async () => { + it('opens confirmation modal when reset offsets button clicked on a STOPPED connector', async () => { + (useConnector as jest.Mock).mockImplementation(() => ({ + data: setConnectorStatus(connector, ConnectorState.STOPPED), + })); renderComponent(); await afterClickDropDownButton(); await waitFor(async () => diff --git a/frontend/src/components/Connect/List/__tests__/List.spec.tsx b/frontend/src/components/Connect/List/__tests__/List.spec.tsx index 3e86448bc..ba4edb082 100644 --- a/frontend/src/components/Connect/List/__tests__/List.spec.tsx +++ b/frontend/src/components/Connect/List/__tests__/List.spec.tsx @@ -12,11 +12,13 @@ import { clusterConnectConnectorPath, clusterConnectorsPath } from 'lib/paths'; import { useConnectors, useDeleteConnector, + useResetConnectorOffsets, useUpdateConnectorState, } from 'lib/hooks/api/kafkaConnect'; const mockedUsedNavigate = jest.fn(); const mockDelete = jest.fn(); +const mockResetOffsets = jest.fn(); jest.mock('react-router-dom', () => ({ ...jest.requireActual('react-router-dom'), @@ -57,7 +59,7 @@ describe('Connectors List', () => { it('renders', async () => { renderComponent(); expect(screen.getByRole('table')).toBeInTheDocument(); - expect(screen.getAllByRole('row').length).toEqual(3); + expect(screen.getAllByRole('row').length).toEqual(4); }); it('opens broker when row clicked', async () => { @@ -129,4 +131,39 @@ describe('Connectors List', () => { expect(cancelButton).not.toBeInTheDocument(); }); }); + + describe('when reset connector offsets modal is open', () => { + beforeEach(() => { + (useConnectors as jest.Mock).mockImplementation(() => ({ + data: connectors, + })); + (useResetConnectorOffsets as jest.Mock).mockImplementation(() => ({ + mutateAsync: mockResetOffsets, + })); + }); + + it('calls resetConnectorOffsets on confirm', async () => { + renderComponent(); + const resetButton = screen.getAllByText('Reset Offsets')[2]; + await waitFor(() => userEvent.click(resetButton)); + + const submitButton = screen.getAllByRole('button', { + name: 'Confirm', + })[0]; + await userEvent.click(submitButton); + expect(mockResetOffsets).toHaveBeenCalledWith(); + }); + + it('closes the modal when cancel button is clicked', async () => { + renderComponent(); + const resetButton = screen.getAllByText('Reset Offsets')[2]; + await waitFor(() => userEvent.click(resetButton)); + + const cancelButton = screen.getAllByRole('button', { + name: 'Cancel', + })[0]; + await waitFor(() => userEvent.click(cancelButton)); + expect(cancelButton).not.toBeInTheDocument(); + }); + }); }); diff --git a/frontend/src/lib/fixtures/kafkaConnect.ts b/frontend/src/lib/fixtures/kafkaConnect.ts index 8a79760e6..4185695ee 100644 --- a/frontend/src/lib/fixtures/kafkaConnect.ts +++ b/frontend/src/lib/fixtures/kafkaConnect.ts @@ -38,6 +38,18 @@ export const connectors: FullConnectorInfo[] = [ tasksCount: 3, failedTasksCount: 1, }, + { + connect: 'third', + name: 'hdfs3-source-connector', + connectorClass: 'FileStreamSource', + type: ConnectorType.SINK, + topics: ['test-topic'], + status: { + state: ConnectorState.STOPPED, + }, + tasksCount: 0, + failedTasksCount: 0, + }, ]; export const connector: Connector = { From 76ae0f1845a22cdbeb9b928ac1fb552048ef73de Mon Sep 17 00:00:00 2001 From: NOZAIS Julien Date: Mon, 7 Oct 2024 15:06:32 +0200 Subject: [PATCH 17/34] fix prettier linter checks --- .../Actions/__tests__/Actions.spec.tsx | 24 ++++++++++++++----- .../components/Connect/List/ActionsCell.tsx | 3 ++- .../Connect/List/__tests__/List.spec.tsx | 2 +- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/frontend/src/components/Connect/Details/Actions/__tests__/Actions.spec.tsx b/frontend/src/components/Connect/Details/Actions/__tests__/Actions.spec.tsx index e2588e8d9..5b746e3ff 100644 --- a/frontend/src/components/Connect/Details/Actions/__tests__/Actions.spec.tsx +++ b/frontend/src/components/Connect/Details/Actions/__tests__/Actions.spec.tsx @@ -89,7 +89,9 @@ describe('Actions', () => { expect(screen.queryByText('Stop')).not.toBeInTheDocument(); await afterClickDropDownButton(); expect(screen.getByText('Reset Offsets')).toBeInTheDocument(); - expect(screen.getByRole('menuitem', { name: 'Reset Offsets' })).toHaveAttribute('aria-disabled'); + expect( + screen.getByRole('menuitem', { name: 'Reset Offsets' }) + ).toHaveAttribute('aria-disabled'); expectActionButtonsExists(); }); @@ -105,7 +107,9 @@ describe('Actions', () => { expect(screen.queryByText('Stop')).not.toBeInTheDocument(); await afterClickDropDownButton(); expect(screen.getByText('Reset Offsets')).toBeInTheDocument(); - expect(screen.getByRole('menuitem', { name: 'Reset Offsets' })).not.toHaveAttribute('aria-disabled'); + expect( + screen.getByRole('menuitem', { name: 'Reset Offsets' }) + ).not.toHaveAttribute('aria-disabled'); expectActionButtonsExists(); }); @@ -121,7 +125,9 @@ describe('Actions', () => { expect(screen.queryByText('Stop')).not.toBeInTheDocument(); await afterClickDropDownButton(); expect(screen.getByText('Reset Offsets')).toBeInTheDocument(); - expect(screen.getByRole('menuitem', { name: 'Reset Offsets' })).toHaveAttribute('aria-disabled'); + expect( + screen.getByRole('menuitem', { name: 'Reset Offsets' }) + ).toHaveAttribute('aria-disabled'); expectActionButtonsExists(); }); @@ -137,7 +143,9 @@ describe('Actions', () => { expect(screen.queryByText('Stop')).not.toBeInTheDocument(); await afterClickDropDownButton(); expect(screen.getByText('Reset Offsets')).toBeInTheDocument(); - expect(screen.getByRole('menuitem', { name: 'Reset Offsets' })).toHaveAttribute('aria-disabled'); + expect( + screen.getByRole('menuitem', { name: 'Reset Offsets' }) + ).toHaveAttribute('aria-disabled'); expectActionButtonsExists(); }); @@ -153,7 +161,9 @@ describe('Actions', () => { expect(screen.getByText('Stop')).toBeInTheDocument(); await afterClickDropDownButton(); expect(screen.getByText('Reset Offsets')).toBeInTheDocument(); - expect(screen.getByRole('menuitem', { name: 'Reset Offsets' })).toHaveAttribute('aria-disabled'); + expect( + screen.getByRole('menuitem', { name: 'Reset Offsets' }) + ).toHaveAttribute('aria-disabled'); expectActionButtonsExists(); }); @@ -180,7 +190,9 @@ describe('Actions', () => { renderComponent(); await afterClickDropDownButton(); await waitFor(async () => - userEvent.click(screen.getByRole('menuitem', { name: 'Reset Offsets' })) + userEvent.click( + screen.getByRole('menuitem', { name: 'Reset Offsets' }) + ) ); expect(screen.getByRole('dialog')).toBeInTheDocument(); }); diff --git a/frontend/src/components/Connect/List/ActionsCell.tsx b/frontend/src/components/Connect/List/ActionsCell.tsx index 720e9be47..4ad50c00e 100644 --- a/frontend/src/components/Connect/List/ActionsCell.tsx +++ b/frontend/src/components/Connect/List/ActionsCell.tsx @@ -173,7 +173,8 @@ const ActionsCell: React.FC> = ({ resource: ResourceType.CONNECT, action: Action.DELETE, value: connect, - }}> + }} + > Delete diff --git a/frontend/src/components/Connect/List/__tests__/List.spec.tsx b/frontend/src/components/Connect/List/__tests__/List.spec.tsx index ba4edb082..194d97246 100644 --- a/frontend/src/components/Connect/List/__tests__/List.spec.tsx +++ b/frontend/src/components/Connect/List/__tests__/List.spec.tsx @@ -131,7 +131,7 @@ describe('Connectors List', () => { expect(cancelButton).not.toBeInTheDocument(); }); }); - + describe('when reset connector offsets modal is open', () => { beforeEach(() => { (useConnectors as jest.Mock).mockImplementation(() => ({ From d95b56ff7ba72b6d3863858c4634c8a1325e8c86 Mon Sep 17 00:00:00 2001 From: NOZAIS Julien Date: Mon, 7 Oct 2024 17:31:52 +0200 Subject: [PATCH 18/34] Improve refresh behaviour on connector actions --- .../src/components/Connect/Details/Actions/Actions.tsx | 8 +------- frontend/src/lib/hooks/api/kafkaConnect.ts | 5 ++++- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/frontend/src/components/Connect/Details/Actions/Actions.tsx b/frontend/src/components/Connect/Details/Actions/Actions.tsx index 379384ab8..6909b4617 100644 --- a/frontend/src/components/Connect/Details/Actions/Actions.tsx +++ b/frontend/src/components/Connect/Details/Actions/Actions.tsx @@ -78,6 +78,7 @@ const Actions: React.FC = () => { return ( Restart @@ -88,7 +89,6 @@ const Actions: React.FC = () => { {connector?.status.state === ConnectorState.RUNNING && ( { {connector?.status.state === ConnectorState.RUNNING && ( { connector?.status.state === ConnectorState.STOPPED) && ( { )} { { { api.updateConnectorState({ ...props, action }), { onSuccess: () => - client.invalidateQueries(['clusters', props.clusterName, 'connectors']), + Promise.all([ + client.invalidateQueries(connectorsKey(props.clusterName)), + client.invalidateQueries(connectorKey(props)), + ]), } ); } From cffa239ec84cc2eb9bba6428ba470985ca02ecfe Mon Sep 17 00:00:00 2001 From: NOZAIS Julien Date: Tue, 29 Oct 2024 17:43:26 +0100 Subject: [PATCH 19/34] refacto using static imports for resetConnectorOffsets actions --- .../io/kafbat/ui/controller/KafkaConnectController.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/api/src/main/java/io/kafbat/ui/controller/KafkaConnectController.java b/api/src/main/java/io/kafbat/ui/controller/KafkaConnectController.java index 209239a7c..328a7353e 100644 --- a/api/src/main/java/io/kafbat/ui/controller/KafkaConnectController.java +++ b/api/src/main/java/io/kafbat/ui/controller/KafkaConnectController.java @@ -3,6 +3,8 @@ import static io.kafbat.ui.model.ConnectorActionDTO.RESTART; import static io.kafbat.ui.model.ConnectorActionDTO.RESTART_ALL_TASKS; import static io.kafbat.ui.model.ConnectorActionDTO.RESTART_FAILED_TASKS; +import static io.kafbat.ui.model.rbac.permission.ConnectAction.RESET_OFFSETS; +import static io.kafbat.ui.model.rbac.permission.ConnectAction.VIEW; import io.kafbat.ui.api.KafkaConnectApi; import io.kafbat.ui.model.ConnectDTO; @@ -290,13 +292,10 @@ private Comparator getConnectorsComparator(ConnectorColumn public Mono> resetConnectorOffsets(String clusterName, String connectName, String connectorName, ServerWebExchange exchange) { - ConnectAction[] connectActions; - - connectActions = new ConnectAction[] { ConnectAction.VIEW, ConnectAction.RESET_OFFSETS }; var context = AccessContext.builder() .cluster(clusterName) - .connectActions(connectName, connectActions) + .connectActions(connectName, VIEW, RESET_OFFSETS) .operationName("resetConnectorOffsets") .operationParams(Map.of(CONNECTOR_NAME, connectorName)) .build(); From e8e4347c7548f137c7f9be5e39e13aca0e518389 Mon Sep 17 00:00:00 2001 From: NOZAIS Julien Date: Tue, 5 Nov 2024 11:03:27 +0100 Subject: [PATCH 20/34] tests: reset stopped connector --- .../io/kafbat/ui/KafkaConnectServiceTests.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/api/src/test/java/io/kafbat/ui/KafkaConnectServiceTests.java b/api/src/test/java/io/kafbat/ui/KafkaConnectServiceTests.java index 56ae1430e..7d00eff4f 100644 --- a/api/src/test/java/io/kafbat/ui/KafkaConnectServiceTests.java +++ b/api/src/test/java/io/kafbat/ui/KafkaConnectServiceTests.java @@ -408,4 +408,20 @@ public void shouldReturn400WhenTryingToCreateConnectorWithExistingName() { .expectStatus() .isBadRequest(); } + + @Test + public void shouldResetConnectorWhenInStoppedState() { + webTestClient.put() + .uri("/api/clusters/{clusterName}/connectors/{connectName}/connectors/{connectorName}/action/STOP", LOCAL, + connectName, connectorName) + .exchange() + .expectStatus().isOk(); + + webTestClient.delete() + .uri("/api/clusters/{clusterName}/connectors/{connectName}/connectors/{connectorName}/offsets", LOCAL, + connectName, connectorName) + .exchange() + .expectStatus().isOk(); + + } } From 67599c1b1e4ba095aabc840eb09d7857fb507f4f Mon Sep 17 00:00:00 2001 From: NOZAIS Julien Date: Tue, 5 Nov 2024 11:04:57 +0100 Subject: [PATCH 21/34] kafka connect API get offsets --- .../resources/swagger/kafka-connect-api.yaml | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/contract/src/main/resources/swagger/kafka-connect-api.yaml b/contract/src/main/resources/swagger/kafka-connect-api.yaml index cfe81bff9..5fa8dc230 100644 --- a/contract/src/main/resources/swagger/kafka-connect-api.yaml +++ b/contract/src/main/resources/swagger/kafka-connect-api.yaml @@ -164,7 +164,22 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/ConnectorOffsetsReset' + $ref: '#/components/schemas/ConnectorOffsetsError' + + get: + tags: + - KafkaConnectClient + summary: Get the offsets for the specified connector + operationId: getConnectorOffsets + parameters: + - in: path + name: connector + required: true + schema: + type: string + responses: + 200: + description: OK /connectors/{connectorName}/status: get: @@ -469,7 +484,7 @@ components: trace: type: string - ConnectorOffsetsReset: + ConnectorOffsetsError: type: object properties: error_code: From 2aebcf6e666e139b19a91e95752936816c37fd39 Mon Sep 17 00:00:00 2001 From: Dugong Date: Tue, 5 Nov 2024 14:27:07 +0000 Subject: [PATCH 22/34] FIX: Type casting errors --- .../java/io/kafbat/ui/service/TopicsService.java | 15 +++++++-------- .../io/kafbat/ui/service/audit/AuditService.java | 4 ++-- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/api/src/main/java/io/kafbat/ui/service/TopicsService.java b/api/src/main/java/io/kafbat/ui/service/TopicsService.java index 015a86838..8c43b77d5 100644 --- a/api/src/main/java/io/kafbat/ui/service/TopicsService.java +++ b/api/src/main/java/io/kafbat/ui/service/TopicsService.java @@ -13,6 +13,7 @@ import io.kafbat.ui.model.InternalLogDirStats; import io.kafbat.ui.model.InternalPartition; import io.kafbat.ui.model.InternalPartitionsOffsets; +import io.kafbat.ui.model.InternalPartitionsOffsets.Offsets; import io.kafbat.ui.model.InternalReplica; import io.kafbat.ui.model.InternalTopic; import io.kafbat.ui.model.InternalTopicConfig; @@ -143,14 +144,12 @@ private Mono getPartitionOffsets(Map - Sets.intersection(earliest.keySet(), latest.keySet()) - .stream() - .map(tp -> - Map.entry(tp, - new InternalPartitionsOffsets.Offsets( - earliest.get(tp), latest.get(tp)))) - .collect(toMap(Map.Entry::getKey, Map.Entry::getValue))) + (earliest, latest) -> (Map) Sets.intersection(earliest.keySet(), latest.keySet()) + .stream() + .map(tp -> Map.entry(tp, + new InternalPartitionsOffsets.Offsets( + earliest.get(tp), latest.get(tp)))) + .collect(toMap(Map.Entry::getKey, Map.Entry::getValue))) .map(InternalPartitionsOffsets::new); } diff --git a/api/src/main/java/io/kafbat/ui/service/audit/AuditService.java b/api/src/main/java/io/kafbat/ui/service/audit/AuditService.java index 22bb77f0b..7ff131613 100644 --- a/api/src/main/java/io/kafbat/ui/service/audit/AuditService.java +++ b/api/src/main/java/io/kafbat/ui/service/audit/AuditService.java @@ -181,11 +181,11 @@ private Mono extractUser(Signal sig) { return sig.getContextView().>get(key) .map(context -> context.getAuthentication().getPrincipal()) - .map(AuditService::extractUser) + .map(AuditService::extractUserFromObject) .switchIfEmpty(Mono.just(UNKNOWN_USER)); } - private static AuthenticatedUser extractUser(Object principal) { + private static AuthenticatedUser extractUserFromObject(Object principal) { if (principal instanceof UserDetails u) { return new AuthenticatedUser(u.getUsername(), Set.of()); } else if (principal instanceof AuthenticatedPrincipal p) { From f7f45919b0b43fe7bf43cd822e228e1c8704be87 Mon Sep 17 00:00:00 2001 From: Dugong Date: Tue, 5 Nov 2024 15:48:05 +0000 Subject: [PATCH 23/34] Test bad request when resetting running connector --- .../java/io/kafbat/ui/KafkaConnectServiceTests.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/api/src/test/java/io/kafbat/ui/KafkaConnectServiceTests.java b/api/src/test/java/io/kafbat/ui/KafkaConnectServiceTests.java index 7d00eff4f..4548cb796 100644 --- a/api/src/test/java/io/kafbat/ui/KafkaConnectServiceTests.java +++ b/api/src/test/java/io/kafbat/ui/KafkaConnectServiceTests.java @@ -408,7 +408,7 @@ public void shouldReturn400WhenTryingToCreateConnectorWithExistingName() { .expectStatus() .isBadRequest(); } - + @Test public void shouldResetConnectorWhenInStoppedState() { webTestClient.put() @@ -424,4 +424,15 @@ public void shouldResetConnectorWhenInStoppedState() { .expectStatus().isOk(); } + + @Test + public void shouldReturn400WhenResettingConnectorInRunningState() { + + webTestClient.delete() + .uri("/api/clusters/{clusterName}/connectors/{connectName}/connectors/{connectorName}/offsets", LOCAL, + connectName, connectorName) + .exchange() + .expectStatus().isBadRequest(); + + } } From 8c648351394e78cb75f60e7d8bec7f01f4fae455 Mon Sep 17 00:00:00 2001 From: Dugong Date: Wed, 6 Nov 2024 14:50:16 +0000 Subject: [PATCH 24/34] Throw controlled BadRequest and NotFound errors for resetConnectorOffsets errors --- .../exception/ConnectorOffsetsResetException.java | 13 +++++++++++++ .../main/java/io/kafbat/ui/exception/ErrorCode.java | 1 + .../io/kafbat/ui/service/KafkaConnectService.java | 13 ++++++++++++- 3 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 api/src/main/java/io/kafbat/ui/exception/ConnectorOffsetsResetException.java diff --git a/api/src/main/java/io/kafbat/ui/exception/ConnectorOffsetsResetException.java b/api/src/main/java/io/kafbat/ui/exception/ConnectorOffsetsResetException.java new file mode 100644 index 000000000..f76feddc3 --- /dev/null +++ b/api/src/main/java/io/kafbat/ui/exception/ConnectorOffsetsResetException.java @@ -0,0 +1,13 @@ +package io.kafbat.ui.exception; + +public class ConnectorOffsetsResetException extends CustomBaseException { + + public ConnectorOffsetsResetException(String message) { + super(message); + } + + @Override + public ErrorCode getErrorCode() { + return ErrorCode.CONNECTOR_OFFSETS_RESET_ERROR; + } +} diff --git a/api/src/main/java/io/kafbat/ui/exception/ErrorCode.java b/api/src/main/java/io/kafbat/ui/exception/ErrorCode.java index a1b499aff..6d4a732e3 100644 --- a/api/src/main/java/io/kafbat/ui/exception/ErrorCode.java +++ b/api/src/main/java/io/kafbat/ui/exception/ErrorCode.java @@ -32,6 +32,7 @@ public enum ErrorCode { TOPIC_ANALYSIS_ERROR(4018, HttpStatus.BAD_REQUEST), FILE_UPLOAD_EXCEPTION(4019, HttpStatus.INTERNAL_SERVER_ERROR), CEL_ERROR(4020, HttpStatus.BAD_REQUEST), + CONNECTOR_OFFSETS_RESET_ERROR(4021, HttpStatus.BAD_REQUEST), ; static { diff --git a/api/src/main/java/io/kafbat/ui/service/KafkaConnectService.java b/api/src/main/java/io/kafbat/ui/service/KafkaConnectService.java index 8b74416d3..2ba1f8662 100644 --- a/api/src/main/java/io/kafbat/ui/service/KafkaConnectService.java +++ b/api/src/main/java/io/kafbat/ui/service/KafkaConnectService.java @@ -7,6 +7,7 @@ import io.kafbat.ui.connect.model.ConnectorStatusConnector; import io.kafbat.ui.connect.model.ConnectorTopics; import io.kafbat.ui.connect.model.TaskStatus; +import io.kafbat.ui.exception.ConnectorOffsetsResetException; import io.kafbat.ui.exception.NotFoundException; import io.kafbat.ui.exception.ValidationException; import io.kafbat.ui.mapper.ClusterMapper; @@ -298,6 +299,16 @@ private ReactiveFailover api(KafkaCluster cluster, String public Mono resetConnectorOffsets(KafkaCluster cluster, String connectName, String connectorName) { return api(cluster, connectName) - .mono(client -> client.resetConnectorOffsets(connectorName)); + .mono(client -> client.resetConnectorOffsets(connectorName)) + .onErrorResume(WebClientResponseException.NotFound.class, + e -> { + throw new NotFoundException("Connector %s not found in %s".formatted(connectorName, connectName)); + }) + .onErrorResume(WebClientResponseException.BadRequest.class, + e -> { + throw new ConnectorOffsetsResetException( + "Failed to reset offsets of connector %s of %s. Make sure it is STOPPED first." + .formatted(connectorName, connectName)); + }); } } From cf7cbdd3d6372709dfd102d69c70a6ad235c857e Mon Sep 17 00:00:00 2001 From: Dugong Date: Wed, 6 Nov 2024 14:51:27 +0000 Subject: [PATCH 25/34] Use Confluent 7.7 and fix tests for resetConnectorOffsets --- .../io/kafbat/ui/AbstractIntegrationTest.java | 2 +- .../kafbat/ui/KafkaConnectServiceTests.java | 27 +++++++++++++++---- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/api/src/test/java/io/kafbat/ui/AbstractIntegrationTest.java b/api/src/test/java/io/kafbat/ui/AbstractIntegrationTest.java index 554387a1a..7f46227d2 100644 --- a/api/src/test/java/io/kafbat/ui/AbstractIntegrationTest.java +++ b/api/src/test/java/io/kafbat/ui/AbstractIntegrationTest.java @@ -38,7 +38,7 @@ public abstract class AbstractIntegrationTest { private static final boolean IS_ARM = System.getProperty("os.arch").contains("arm") || System.getProperty("os.arch").contains("aarch64"); - private static final String CONFLUENT_PLATFORM_VERSION = IS_ARM ? "7.2.1.arm64" : "7.2.1"; + private static final String CONFLUENT_PLATFORM_VERSION = IS_ARM ? "7.7.1.arm64" : "7.7.1"; public static final KafkaContainer kafka = new KafkaContainer( DockerImageName.parse("confluentinc/cp-kafka").withTag(CONFLUENT_PLATFORM_VERSION)) diff --git a/api/src/test/java/io/kafbat/ui/KafkaConnectServiceTests.java b/api/src/test/java/io/kafbat/ui/KafkaConnectServiceTests.java index 4548cb796..5035d7dcc 100644 --- a/api/src/test/java/io/kafbat/ui/KafkaConnectServiceTests.java +++ b/api/src/test/java/io/kafbat/ui/KafkaConnectServiceTests.java @@ -411,15 +411,24 @@ public void shouldReturn400WhenTryingToCreateConnectorWithExistingName() { @Test public void shouldResetConnectorWhenInStoppedState() { - webTestClient.put() - .uri("/api/clusters/{clusterName}/connectors/{connectName}/connectors/{connectorName}/action/STOP", LOCAL, - connectName, connectorName) + + webTestClient.post() + .uri("/api/clusters/{clusterName}/connects/{connectName}/connectors/{connectorName}/action/STOP", + LOCAL, connectName, connectorName) .exchange() .expectStatus().isOk(); + webTestClient.get() + .uri("/api/clusters/{clusterName}/connects/{connectName}/connectors/{connectorName}", + LOCAL, connectName, connectorName) + .exchange() + .expectStatus().isOk() + .expectBody(ConnectorDTO.class) + .value(connector -> assertThat(connector.getStatus().getState()).isEqualTo(ConnectorStateDTO.STOPPED)); + webTestClient.delete() - .uri("/api/clusters/{clusterName}/connectors/{connectName}/connectors/{connectorName}/offsets", LOCAL, - connectName, connectorName) + .uri("/api/clusters/{clusterName}/connects/{connectName}/connectors/{connectorName}/offsets", + LOCAL, connectName, connectorName) .exchange() .expectStatus().isOk(); @@ -428,6 +437,14 @@ public void shouldResetConnectorWhenInStoppedState() { @Test public void shouldReturn400WhenResettingConnectorInRunningState() { + webTestClient.get() + .uri("/api/clusters/{clusterName}/connects/{connectName}/connectors/{connectorName}", + LOCAL, connectName, connectorName) + .exchange() + .expectStatus().isOk() + .expectBody(ConnectorDTO.class) + .value(connector -> assertThat(connector.getStatus().getState()).isEqualTo(ConnectorStateDTO.RUNNING)); + webTestClient.delete() .uri("/api/clusters/{clusterName}/connectors/{connectName}/connectors/{connectorName}/offsets", LOCAL, connectName, connectorName) From 8f5915cb120ef69bea8de6bcaa391c17dbe48e56 Mon Sep 17 00:00:00 2001 From: Dugong Date: Wed, 6 Nov 2024 14:52:35 +0000 Subject: [PATCH 26/34] typo in test --- api/src/test/java/io/kafbat/ui/KafkaConnectServiceTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/test/java/io/kafbat/ui/KafkaConnectServiceTests.java b/api/src/test/java/io/kafbat/ui/KafkaConnectServiceTests.java index 5035d7dcc..7c24053b7 100644 --- a/api/src/test/java/io/kafbat/ui/KafkaConnectServiceTests.java +++ b/api/src/test/java/io/kafbat/ui/KafkaConnectServiceTests.java @@ -446,7 +446,7 @@ public void shouldReturn400WhenResettingConnectorInRunningState() { .value(connector -> assertThat(connector.getStatus().getState()).isEqualTo(ConnectorStateDTO.RUNNING)); webTestClient.delete() - .uri("/api/clusters/{clusterName}/connectors/{connectName}/connectors/{connectorName}/offsets", LOCAL, + .uri("/api/clusters/{clusterName}/connects/{connectName}/connectors/{connectorName}/offsets", LOCAL, connectName, connectorName) .exchange() .expectStatus().isBadRequest(); From 62598e45f324856ed418cbba159d59b1ef769420 Mon Sep 17 00:00:00 2001 From: Dugong Date: Wed, 6 Nov 2024 16:37:27 +0000 Subject: [PATCH 27/34] retry on kafka connect test setup failure --- .../kafbat/ui/KafkaConnectServiceTests.java | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/api/src/test/java/io/kafbat/ui/KafkaConnectServiceTests.java b/api/src/test/java/io/kafbat/ui/KafkaConnectServiceTests.java index 7c24053b7..55beeb1eb 100644 --- a/api/src/test/java/io/kafbat/ui/KafkaConnectServiceTests.java +++ b/api/src/test/java/io/kafbat/ui/KafkaConnectServiceTests.java @@ -17,6 +17,7 @@ import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -44,6 +45,26 @@ public class KafkaConnectServiceTests extends AbstractIntegrationTest { @BeforeEach public void setUp() { + + int limit = 5; + int tries = 0; + boolean failed = false; + + do { + try { + TimeUnit.SECONDS.sleep(1); + webTestClient.get() + .uri("/api/clusters/{clusterName}/connects/{connectName}/connectors", LOCAL, + connectName) + .exchange() + .expectStatus().isOk(); + } catch (Exception e) { + failed = true; + System.out.println("failed to retrieve connectors: " + tries); + } + tries++; + } while (failed == true && tries < limit); + webTestClient.post() .uri("/api/clusters/{clusterName}/connects/{connectName}/connectors", LOCAL, connectName) .bodyValue(new NewConnectorDTO() From 793c0a579272e9e609b9934c16cb8b846528960c Mon Sep 17 00:00:00 2001 From: Dugong Date: Thu, 7 Nov 2024 17:35:23 +0000 Subject: [PATCH 28/34] Handle error 500 by repeating the kafka connect test setUp --- .../kafbat/ui/KafkaConnectServiceTests.java | 61 +++++++++++-------- 1 file changed, 36 insertions(+), 25 deletions(-) diff --git a/api/src/test/java/io/kafbat/ui/KafkaConnectServiceTests.java b/api/src/test/java/io/kafbat/ui/KafkaConnectServiceTests.java index 55beeb1eb..494e950fa 100644 --- a/api/src/test/java/io/kafbat/ui/KafkaConnectServiceTests.java +++ b/api/src/test/java/io/kafbat/ui/KafkaConnectServiceTests.java @@ -24,6 +24,8 @@ import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.ParameterizedTypeReference; +import org.springframework.http.HttpStatus; +import org.springframework.test.web.reactive.server.ExchangeResult; import org.springframework.test.web.reactive.server.WebTestClient; @Slf4j @@ -51,34 +53,35 @@ public void setUp() { boolean failed = false; do { - try { - TimeUnit.SECONDS.sleep(1); - webTestClient.get() - .uri("/api/clusters/{clusterName}/connects/{connectName}/connectors", LOCAL, - connectName) - .exchange() - .expectStatus().isOk(); - } catch (Exception e) { - failed = true; - System.out.println("failed to retrieve connectors: " + tries); - } + + ExchangeResult result = webTestClient.post() + .uri("/api/clusters/{clusterName}/connects/{connectName}/connectors", LOCAL, connectName) + .bodyValue(new NewConnectorDTO() + .name(connectorName) + .config(Map.of( + "connector.class", "org.apache.kafka.connect.file.FileStreamSinkConnector", + "tasks.max", "1", + "topics", "output-topic", + "file", "/tmp/test", + "test.password", "test-credentials"))) + .exchange() + .expectBody() + .returnResult(); + + // Kafka Connect returns an error 500 during occasional rebalances + failed = result.getStatus() == HttpStatus.INTERNAL_SERVER_ERROR; tries++; + + if (failed) { + System.out.println("Failed to setUp connector %s time(s), got status: %s".formatted(tries, result.getStatus())); + try { + TimeUnit.SECONDS.sleep(1); + } catch (Exception e) { + System.out.println("Sleep got interrupted"); + } + } } while (failed == true && tries < limit); - webTestClient.post() - .uri("/api/clusters/{clusterName}/connects/{connectName}/connectors", LOCAL, connectName) - .bodyValue(new NewConnectorDTO() - .name(connectorName) - .config(Map.of( - "connector.class", "org.apache.kafka.connect.file.FileStreamSinkConnector", - "tasks.max", "1", - "topics", "output-topic", - "file", "/tmp/test", - "test.password", "test-credentials" - )) - ) - .exchange() - .expectStatus().isOk(); } @AfterEach @@ -433,6 +436,14 @@ public void shouldReturn400WhenTryingToCreateConnectorWithExistingName() { @Test public void shouldResetConnectorWhenInStoppedState() { + webTestClient.get() + .uri("/api/clusters/{clusterName}/connects/{connectName}/connectors/{connectorName}", + LOCAL, connectName, connectorName) + .exchange() + .expectStatus().isOk() + .expectBody(ConnectorDTO.class) + .value(connector -> assertThat(connector.getStatus().getState()).isEqualTo(ConnectorStateDTO.RUNNING)); + webTestClient.post() .uri("/api/clusters/{clusterName}/connects/{connectName}/connectors/{connectorName}/action/STOP", LOCAL, connectName, connectorName) From 4a499e4c15874dc7021dc1f63922fe6f1fc1534c Mon Sep 17 00:00:00 2001 From: Dugong Date: Fri, 8 Nov 2024 14:56:34 +0000 Subject: [PATCH 29/34] Optimistic creation of test connectors. Check existence and ignore create errors. --- .../kafbat/ui/KafkaConnectServiceTests.java | 57 ++++++++----------- 1 file changed, 24 insertions(+), 33 deletions(-) diff --git a/api/src/test/java/io/kafbat/ui/KafkaConnectServiceTests.java b/api/src/test/java/io/kafbat/ui/KafkaConnectServiceTests.java index 494e950fa..00a7f0866 100644 --- a/api/src/test/java/io/kafbat/ui/KafkaConnectServiceTests.java +++ b/api/src/test/java/io/kafbat/ui/KafkaConnectServiceTests.java @@ -48,40 +48,31 @@ public class KafkaConnectServiceTests extends AbstractIntegrationTest { @BeforeEach public void setUp() { - int limit = 5; - int tries = 0; - boolean failed = false; - - do { - - ExchangeResult result = webTestClient.post() - .uri("/api/clusters/{clusterName}/connects/{connectName}/connectors", LOCAL, connectName) - .bodyValue(new NewConnectorDTO() - .name(connectorName) - .config(Map.of( - "connector.class", "org.apache.kafka.connect.file.FileStreamSinkConnector", - "tasks.max", "1", - "topics", "output-topic", - "file", "/tmp/test", - "test.password", "test-credentials"))) - .exchange() - .expectBody() - .returnResult(); - - // Kafka Connect returns an error 500 during occasional rebalances - failed = result.getStatus() == HttpStatus.INTERNAL_SERVER_ERROR; - tries++; - - if (failed) { - System.out.println("Failed to setUp connector %s time(s), got status: %s".formatted(tries, result.getStatus())); - try { - TimeUnit.SECONDS.sleep(1); - } catch (Exception e) { - System.out.println("Sleep got interrupted"); - } - } - } while (failed == true && tries < limit); + ExchangeResult creationResult = webTestClient.post() + .uri("/api/clusters/{clusterName}/connects/{connectName}/connectors", LOCAL, connectName) + .bodyValue(new NewConnectorDTO() + .name(connectorName) + .config(Map.of( + "connector.class", "org.apache.kafka.connect.file.FileStreamSinkConnector", + "tasks.max", "1", + "topics", "output-topic", + "file", "/tmp/test", + "test.password", "test-credentials"))) + .exchange() + .expectBody() + .returnResult(); + + webTestClient.get() + .uri("/api/clusters/{clusterName}/connects/{connectName}/connectors/{connectorName}", + LOCAL, connectName, connectorName) + .exchange() + .expectStatus().isOk(); + // Kafka Connect may return transient HTTP 500 errors during rebalances + if (creationResult.getStatus() != HttpStatus.OK) { + log.warn( + "Ignoring a transient error while setting up the tested connector, because it has been created anyway."); + } } @AfterEach From 035f2b3a4e289667a460f7ba8356c739bdf53f58 Mon Sep 17 00:00:00 2001 From: NOZAIS Julien Date: Fri, 8 Nov 2024 16:12:38 +0100 Subject: [PATCH 30/34] cleanup --- api/src/test/java/io/kafbat/ui/KafkaConnectServiceTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/api/src/test/java/io/kafbat/ui/KafkaConnectServiceTests.java b/api/src/test/java/io/kafbat/ui/KafkaConnectServiceTests.java index 00a7f0866..3ef3907d5 100644 --- a/api/src/test/java/io/kafbat/ui/KafkaConnectServiceTests.java +++ b/api/src/test/java/io/kafbat/ui/KafkaConnectServiceTests.java @@ -17,7 +17,6 @@ import java.util.List; import java.util.Map; import java.util.UUID; -import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; From 0d8ba5416b376f09b7039516b0ef73da3947411b Mon Sep 17 00:00:00 2001 From: Dugong Date: Fri, 3 Jan 2025 11:11:43 +0000 Subject: [PATCH 31/34] adapt to connect retry mechanism use the proper return type for the client interface fix stopConnector override with new retry mechanism --- .../ui/client/RetryingKafkaConnectClient.java | 23 ++++++++++++++++--- .../kafbat/ui/KafkaConnectServiceTests.java | 12 +--------- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/api/src/main/java/io/kafbat/ui/client/RetryingKafkaConnectClient.java b/api/src/main/java/io/kafbat/ui/client/RetryingKafkaConnectClient.java index 0f2185bf2..df2da3e55 100644 --- a/api/src/main/java/io/kafbat/ui/client/RetryingKafkaConnectClient.java +++ b/api/src/main/java/io/kafbat/ui/client/RetryingKafkaConnectClient.java @@ -233,14 +233,19 @@ public Mono pauseConnector(String connectorName) throws WebClientResponseE return withRetryOnConflictOrRebalance(super.pauseConnector(connectorName)); } + @Override + public Mono> pauseConnectorWithHttpInfo(String connectorName) throws WebClientResponseException { + return withRetryOnConflictOrRebalance(super.pauseConnectorWithHttpInfo(connectorName)); + } + @Override public Mono stopConnector(String connectorName) throws WebClientResponseException { - return withRetryOnConflict(super.stopConnector(connectorName)); + return withRetryOnConflictOrRebalance(super.stopConnector(connectorName)); } @Override - public Mono> pauseConnectorWithHttpInfo(String connectorName) throws WebClientResponseException { - return withRetryOnConflictOrRebalance(super.pauseConnectorWithHttpInfo(connectorName)); + public Mono> stopConnectorWithHttpInfo(String connectorName) throws WebClientResponseException { + return withRetryOnConflictOrRebalance(super.stopConnectorWithHttpInfo(connectorName)); } @Override @@ -266,6 +271,18 @@ public Mono> restartConnectorTaskWithHttpInfo(String connec return withRetryOnConflictOrRebalance(super.restartConnectorTaskWithHttpInfo(connectorName, taskId)); } + @Override + public Mono resetConnectorOffsets(String connectorName) + throws WebClientResponseException { + return withRetryOnConflictOrRebalance(super.resetConnectorOffsets(connectorName)); + } + + @Override + public Mono> resetConnectorOffsetsWithHttpInfo(String connectorName) + throws WebClientResponseException { + return withRetryOnConflictOrRebalance(super.resetConnectorOffsetsWithHttpInfo(connectorName)); + } + @Override public Mono resumeConnector(String connectorName) throws WebClientResponseException { return withRetryOnRebalance(super.resumeConnector(connectorName)); diff --git a/api/src/test/java/io/kafbat/ui/KafkaConnectServiceTests.java b/api/src/test/java/io/kafbat/ui/KafkaConnectServiceTests.java index 1415f8d69..b12d5e78b 100644 --- a/api/src/test/java/io/kafbat/ui/KafkaConnectServiceTests.java +++ b/api/src/test/java/io/kafbat/ui/KafkaConnectServiceTests.java @@ -59,20 +59,10 @@ public void setUp() { "file", "/tmp/test", "test.password", "test-credentials"))) .exchange() + .expectStatus().isOk() .expectBody() .returnResult(); - webTestClient.get() - .uri("/api/clusters/{clusterName}/connects/{connectName}/connectors/{connectorName}", - LOCAL, connectName, connectorName) - .exchange() - .expectStatus().isOk(); - - // Kafka Connect may return transient HTTP 500 errors during rebalances - if (creationResult.getStatus() != HttpStatus.OK) { - log.warn( - "Ignoring a transient error while setting up the tested connector, because it has been created anyway."); - } } @AfterEach From e802bf1b6c83443f15defc0a27864446723e5182 Mon Sep 17 00:00:00 2001 From: Dugong Date: Fri, 3 Jan 2025 13:51:57 +0000 Subject: [PATCH 32/34] rollback test workaround leftovers --- .../test/java/io/kafbat/ui/KafkaConnectServiceTests.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/api/src/test/java/io/kafbat/ui/KafkaConnectServiceTests.java b/api/src/test/java/io/kafbat/ui/KafkaConnectServiceTests.java index b12d5e78b..c5fbb14b4 100644 --- a/api/src/test/java/io/kafbat/ui/KafkaConnectServiceTests.java +++ b/api/src/test/java/io/kafbat/ui/KafkaConnectServiceTests.java @@ -48,7 +48,7 @@ public class KafkaConnectServiceTests extends AbstractIntegrationTest { @BeforeEach public void setUp() { - ExchangeResult creationResult = webTestClient.post() + webTestClient.post() .uri("/api/clusters/{clusterName}/connects/{connectName}/connectors", LOCAL, connectName) .bodyValue(new NewConnectorDTO() .name(connectorName) @@ -59,9 +59,7 @@ public void setUp() { "file", "/tmp/test", "test.password", "test-credentials"))) .exchange() - .expectStatus().isOk() - .expectBody() - .returnResult(); + .expectStatus().isOk(); } From 0e1bdb9bb81c70bd5be7482ce5ae956c19085579 Mon Sep 17 00:00:00 2001 From: Dugong Date: Fri, 3 Jan 2025 13:52:32 +0000 Subject: [PATCH 33/34] rollback forced typing leftovers --- .../main/java/io/kafbat/ui/service/TopicsService.java | 11 ++++++----- .../java/io/kafbat/ui/service/audit/AuditService.java | 4 ++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/api/src/main/java/io/kafbat/ui/service/TopicsService.java b/api/src/main/java/io/kafbat/ui/service/TopicsService.java index 8c43b77d5..7b5ae20fb 100644 --- a/api/src/main/java/io/kafbat/ui/service/TopicsService.java +++ b/api/src/main/java/io/kafbat/ui/service/TopicsService.java @@ -13,7 +13,6 @@ import io.kafbat.ui.model.InternalLogDirStats; import io.kafbat.ui.model.InternalPartition; import io.kafbat.ui.model.InternalPartitionsOffsets; -import io.kafbat.ui.model.InternalPartitionsOffsets.Offsets; import io.kafbat.ui.model.InternalReplica; import io.kafbat.ui.model.InternalTopic; import io.kafbat.ui.model.InternalTopicConfig; @@ -144,11 +143,13 @@ private Mono getPartitionOffsets(Map (Map) Sets.intersection(earliest.keySet(), latest.keySet()) + (earliest, latest) -> + Sets.intersection(earliest.keySet(), latest.keySet()) .stream() - .map(tp -> Map.entry(tp, - new InternalPartitionsOffsets.Offsets( - earliest.get(tp), latest.get(tp)))) + .map(tp -> + Map.entry(tp, + new InternalPartitionsOffsets.Offsets( + earliest.get(tp), latest.get(tp)))) .collect(toMap(Map.Entry::getKey, Map.Entry::getValue))) .map(InternalPartitionsOffsets::new); } diff --git a/api/src/main/java/io/kafbat/ui/service/audit/AuditService.java b/api/src/main/java/io/kafbat/ui/service/audit/AuditService.java index 7ff131613..22bb77f0b 100644 --- a/api/src/main/java/io/kafbat/ui/service/audit/AuditService.java +++ b/api/src/main/java/io/kafbat/ui/service/audit/AuditService.java @@ -181,11 +181,11 @@ private Mono extractUser(Signal sig) { return sig.getContextView().>get(key) .map(context -> context.getAuthentication().getPrincipal()) - .map(AuditService::extractUserFromObject) + .map(AuditService::extractUser) .switchIfEmpty(Mono.just(UNKNOWN_USER)); } - private static AuthenticatedUser extractUserFromObject(Object principal) { + private static AuthenticatedUser extractUser(Object principal) { if (principal instanceof UserDetails u) { return new AuthenticatedUser(u.getUsername(), Set.of()); } else if (principal instanceof AuthenticatedPrincipal p) { From fbb6ddd7a1caced69a6e0dacea4a91c02e87d2df Mon Sep 17 00:00:00 2001 From: NOZAIS Julien Date: Fri, 3 Jan 2025 17:27:29 +0100 Subject: [PATCH 34/34] fix indent --- .../java/io/kafbat/ui/service/TopicsService.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/api/src/main/java/io/kafbat/ui/service/TopicsService.java b/api/src/main/java/io/kafbat/ui/service/TopicsService.java index 7b5ae20fb..015a86838 100644 --- a/api/src/main/java/io/kafbat/ui/service/TopicsService.java +++ b/api/src/main/java/io/kafbat/ui/service/TopicsService.java @@ -144,13 +144,13 @@ private Mono getPartitionOffsets(Map - Sets.intersection(earliest.keySet(), latest.keySet()) - .stream() - .map(tp -> - Map.entry(tp, - new InternalPartitionsOffsets.Offsets( - earliest.get(tp), latest.get(tp)))) - .collect(toMap(Map.Entry::getKey, Map.Entry::getValue))) + Sets.intersection(earliest.keySet(), latest.keySet()) + .stream() + .map(tp -> + Map.entry(tp, + new InternalPartitionsOffsets.Offsets( + earliest.get(tp), latest.get(tp)))) + .collect(toMap(Map.Entry::getKey, Map.Entry::getValue))) .map(InternalPartitionsOffsets::new); }