-
Notifications
You must be signed in to change notification settings - Fork 10
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
add test for simulating the problem for continuous receive
- Loading branch information
Showing
13 changed files
with
445 additions
and
9 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
97 changes: 97 additions & 0 deletions
97
src/test-e2e/kotlin/io/github/nomisRev/kafka/e2e/setup/KafkaApplicationUnderTest.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,97 @@ | ||
| package io.github.nomisRev.kafka.e2e.setup | ||
|
|
||
| import com.trendyol.stove.testing.e2e.system.abstractions.ApplicationUnderTest | ||
| import io.github.nomisRev.kafka.e2e.setup.example.KafkaTestShared | ||
| import io.github.nomisRev.kafka.e2e.setup.example.ReceiveMethod | ||
| import io.github.nomisRev.kafka.e2e.setup.example.StoveKafkaValueDeserializer | ||
| import io.github.nomisRev.kafka.e2e.setup.example.StoveKafkaValueSerializer | ||
| import io.github.nomisRev.kafka.publisher.KafkaPublisher | ||
| import io.github.nomisRev.kafka.publisher.PublisherSettings | ||
| import io.github.nomisRev.kafka.receiver.CommitStrategy | ||
| import io.github.nomisRev.kafka.receiver.KafkaReceiver | ||
| import io.github.nomisRev.kafka.receiver.ReceiverSettings | ||
| import org.apache.kafka.clients.admin.AdminClient | ||
| import org.apache.kafka.clients.admin.AdminClientConfig | ||
| import org.apache.kafka.clients.admin.NewTopic | ||
| import org.apache.kafka.clients.consumer.ConsumerConfig | ||
| import org.apache.kafka.clients.producer.ProducerConfig | ||
| import org.apache.kafka.common.serialization.StringDeserializer | ||
| import org.apache.kafka.common.serialization.StringSerializer | ||
| import java.util.* | ||
| import kotlin.time.Duration.Companion.seconds | ||
|
|
||
| /** | ||
| * Stove's Kafka application under test implementation | ||
| */ | ||
| class KafkaApplicationUnderTest : ApplicationUnderTest<Unit> { | ||
| private lateinit var client: AdminClient | ||
| private val consumers: MutableList<AutoCloseable> = mutableListOf() | ||
|
|
||
| override suspend fun start(configurations: List<String>) { | ||
| val bootstrapServers = configurations.first { it.contains("kafka.servers", true) }.split('=')[1] | ||
| val interceptorClass = configurations.first { it.contains("kafka.interceptor-classes", true) }.split('=')[1] | ||
| val receiveMethod = configurations.first { it.contains("kafka.receive-method", true) }.split('=')[1] | ||
| client = createAdminClient(bootstrapServers) | ||
| createTopics(client) | ||
| startConsumers(bootstrapServers, interceptorClass, ReceiveMethod.from(receiveMethod)) | ||
| } | ||
|
|
||
| private fun createAdminClient(bootstrapServers: String): AdminClient { | ||
| return mapOf<String, Any>( | ||
| AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG to bootstrapServers | ||
| ).let { AdminClient.create(it) } | ||
| } | ||
|
|
||
| private fun createTopics(client: AdminClient) { | ||
| val newTopics = KafkaTestShared.topics.flatMap { | ||
| listOf(it.topic, it.retryTopic, it.deadLetterTopic) | ||
| }.map { NewTopic(it, 1, 1) } | ||
| client.createTopics(newTopics).all().get() | ||
| } | ||
|
|
||
| private fun startConsumers(bootStrapServers: String, interceptorClass: String, receiveMethod: ReceiveMethod) { | ||
| val (publisher, receiver) = createPublisherAndReceiver(interceptorClass, bootStrapServers) | ||
| val configuredConsumers = KafkaTestShared.consumers(receiver, publisher, receiveMethod) | ||
| configuredConsumers.forEach { it.start() } | ||
| consumers.addAll(configuredConsumers) | ||
| } | ||
|
|
||
| private fun createPublisherAndReceiver( | ||
| interceptorClass: String, bootStrapServers: String | ||
| ): Pair<KafkaPublisher<String, Any>, KafkaReceiver<String, Any>> { | ||
| val consumerSettings = mapOf( | ||
| ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG to "2000", | ||
| ConsumerConfig.ALLOW_AUTO_CREATE_TOPICS_CONFIG to "true", // Expected to be created by the client | ||
| ConsumerConfig.AUTO_OFFSET_RESET_CONFIG to "earliest", | ||
| ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG to listOf(interceptorClass) | ||
| ) | ||
|
|
||
| val receiverSettings = ReceiverSettings(bootstrapServers = bootStrapServers, | ||
| valueDeserializer = StoveKafkaValueDeserializer(), | ||
| keyDeserializer = StringDeserializer(), | ||
| groupId = "stove-application-consumers", | ||
| commitStrategy = CommitStrategy.ByTime(2.seconds), | ||
| pollTimeout = 1.seconds, | ||
| properties = Properties().apply { | ||
| putAll(consumerSettings) | ||
| }) | ||
|
|
||
| val producerSettings = PublisherSettings<String, Any>(bootStrapServers, | ||
| StringSerializer(), | ||
| StoveKafkaValueSerializer(), | ||
| properties = Properties().apply { | ||
| put( | ||
| ProducerConfig.INTERCEPTOR_CLASSES_CONFIG, listOf(interceptorClass) | ||
| ) | ||
| }) | ||
|
|
||
| val publisher = KafkaPublisher(producerSettings) | ||
| val receiver = KafkaReceiver(receiverSettings) | ||
| return Pair(publisher, receiver) | ||
| } | ||
|
|
||
| override suspend fun stop() { | ||
| client.close() | ||
| consumers.forEach { it.close() } | ||
| } | ||
| } |
26 changes: 26 additions & 0 deletions
26
src/test-e2e/kotlin/io/github/nomisRev/kafka/e2e/setup/ProjectConfig.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| package io.github.nomisRev.kafka.e2e.setup | ||
|
|
||
| import com.trendyol.stove.testing.e2e.standalone.kafka.KafkaSystemOptions | ||
| import com.trendyol.stove.testing.e2e.standalone.kafka.kafka | ||
| import com.trendyol.stove.testing.e2e.system.TestSystem | ||
| import io.kotest.core.config.AbstractProjectConfig | ||
|
|
||
| class ProjectConfig : AbstractProjectConfig() { | ||
| override suspend fun beforeProject(): Unit = TestSystem() | ||
| .with { | ||
| kafka { | ||
| KafkaSystemOptions( | ||
| configureExposedConfiguration = { cfg -> | ||
| listOf( | ||
| "kafka.servers=${cfg.bootstrapServers}", | ||
| "kafka.interceptor-classes=${cfg.interceptorClass}", | ||
| "kafka.receive-method=kotlin-kafka" // here we can change to: 'kotlin-kafka' or 'traditional' | ||
| ) | ||
| } | ||
| ) | ||
| } | ||
| applicationUnderTest(KafkaApplicationUnderTest()) | ||
| }.run() | ||
|
|
||
| override suspend fun afterProject(): Unit = TestSystem.stop() | ||
| } |
113 changes: 113 additions & 0 deletions
113
src/test-e2e/kotlin/io/github/nomisRev/kafka/e2e/setup/example/ConsumerSupervisor.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,113 @@ | ||
| package io.github.nomisRev.kafka.e2e.setup.example | ||
|
|
||
| import io.github.nomisRev.kafka.e2e.setup.example.KafkaTestShared.TopicDefinition | ||
| import io.github.nomisRev.kafka.publisher.KafkaPublisher | ||
| import io.github.nomisRev.kafka.receiver.KafkaReceiver | ||
| import kotlinx.coroutines.* | ||
| import kotlinx.coroutines.flow.flattenConcat | ||
| import org.apache.kafka.clients.consumer.ConsumerRecord | ||
| import org.apache.kafka.clients.producer.ProducerRecord | ||
| import java.time.Duration | ||
|
|
||
| /** | ||
| * Supervisor that uses KafkaReceiver to retrieve messages from Kafka and handle them accordingly | ||
| */ | ||
| abstract class ConsumerSupervisor<K, V>( | ||
| private val receiver: KafkaReceiver<K, V>, | ||
| private val publisher: KafkaPublisher<K, V>, | ||
| private val receiveMethod: ReceiveMethod | ||
| ) : AutoCloseable { | ||
| private val scope = CoroutineScope(Dispatchers.IO + SupervisorJob()) | ||
| private val logger = org.slf4j.LoggerFactory.getLogger(javaClass) | ||
|
|
||
| abstract val topicDefinition: TopicDefinition | ||
|
|
||
| /** | ||
| * Here we start the consumer | ||
| * We can use either KotlinKafka receiver or traditional while(true) loop to receive messages | ||
| * Traditional while(true) loop is successful in receiving messages continuously | ||
| * KotlinKafka receiver, continuously receives messages | ||
| */ | ||
| fun start() = when (receiveMethod) { | ||
| ReceiveMethod.KOTLIN_KAFKA_RECEIVE -> kotlinKafkaReceive() | ||
| ReceiveMethod.TRADITIONAL_RECEIVE -> traditionalReceive() | ||
| } | ||
|
|
||
| @OptIn(ExperimentalCoroutinesApi::class) | ||
| private fun kotlinKafkaReceive() { | ||
| scope.launch { | ||
| receiver.receiveAutoAck( | ||
| listOf( | ||
| topicDefinition.topic, | ||
| topicDefinition.retryTopic, | ||
| topicDefinition.deadLetterTopic | ||
| ) | ||
| ).flattenConcat() | ||
| .collect { message -> | ||
| logger.info("Message RECEIVED on the application side with KotlinKafka receiver: ${message.value()}") | ||
| received(message) // expected to receive the messages continuously? | ||
| } | ||
| } | ||
| } | ||
|
|
||
| private fun traditionalReceive() { | ||
| scope.launch { | ||
| receiver.withConsumer { consumer -> | ||
| consumer.subscribe( | ||
| listOf( | ||
| topicDefinition.topic, | ||
| topicDefinition.retryTopic, | ||
| topicDefinition.deadLetterTopic | ||
| ) | ||
| ) | ||
| while (isActive) { | ||
| val records = consumer.poll(Duration.ofMillis(500)) | ||
| records.forEach { record -> | ||
| logger.info("Message RECEIVED on the application side with traditional while(true) loop: ${record.value()}") | ||
| received(record) { | ||
| consumer.commitAsync() | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
|
|
||
| abstract suspend fun consume(record: ConsumerRecord<K, V>) | ||
|
|
||
| protected open suspend fun handleError(message: ConsumerRecord<K, V>, e: Exception) { | ||
| logger.error("Failed to process message: $message", e) | ||
| } | ||
|
|
||
| private suspend fun received(message: ConsumerRecord<K, V>, onSuccess: (ConsumerRecord<K, V>) -> Unit = { }) { | ||
| try { | ||
| consume(message) | ||
| onSuccess(message) | ||
| logger.info("Message COMMITTED on the application side: ${message.value()}") | ||
| } catch (e: Exception) { | ||
| handleError(message, e) | ||
| logger.warn("CONSUMER GOT an ERROR on the application side, exception: $e") | ||
| val record = ProducerRecord<K, V>( | ||
| topicDefinition.deadLetterTopic, | ||
| message.partition(), | ||
| message.key(), | ||
| message.value(), | ||
| message.headers() | ||
| ) | ||
| try { | ||
| publisher.publishScope { offer(record) } | ||
| } catch (e: Exception) { | ||
| logger.error("Failed to publish message to dead letter topic: $message", e) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| override fun close(): Unit = runBlocking { | ||
| try { | ||
| scope.cancel() | ||
| } catch (e: Exception) { | ||
| logger.error("Failed to stop consuming", e) | ||
| } | ||
| } | ||
| } |
5 changes: 5 additions & 0 deletions
5
src/test-e2e/kotlin/io/github/nomisRev/kafka/e2e/setup/example/DomainEvents.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| package io.github.nomisRev.kafka.e2e.setup.example | ||
|
|
||
| object DomainEvents { | ||
| data class ProductCreated(val productId: String) | ||
| } |
Oops, something went wrong.