Debezium streaming integration

api/pom.xml

@@ -73,6 +73,26 @@
             <artifactId>commons-dbcp2</artifactId>
         </dependency>
 
+        <!--        <dependency>-->
+        <!--            <groupId>io.debezium</groupId>-->
+        <!--            <artifactId>debezium-embedded</artifactId>-->
+        <!--        </dependency>-->
+
+        <!--        <dependency>-->
+        <!--            <groupId>io.debezium</groupId>-->
+        <!--            <artifactId>debezium-api</artifactId>-->
+        <!--        </dependency>-->
+
+        <!--        <dependency>-->
+        <!--            <groupId>io.debezium</groupId>-->
+        <!--            <artifactId>debezium-connector-mysql</artifactId>-->
+        <!--        </dependency>-->
+
+        <dependency>
+            <groupId>org.openmrs.module</groupId>
+            <artifactId>dbevent-api</artifactId>
+        </dependency>
+
     </dependencies>
 
     <build>
@@ -125,4 +145,4 @@
 
     </build>
 
-</project>
+</project>

api/src/main/java/org/openmrs/module/mambacore/api/dao/FlattenDatabaseDao.java

@@ -11,5 +11,13 @@
 
 public interface FlattenDatabaseDao {
 
+	/**
+	 * Deploy MambaETL stored procedures
+	 */
 	void deployMambaEtl();
+
+	/**
+	 * Stream in database changes using Debezium
+	 */
+	void streamInDatabaseChanges();
 }

api/src/main/java/org/openmrs/module/mambacore/api/dao/impl/JdbcFlattenDatabaseDao.java

@@ -1,7 +1,11 @@
 package org.openmrs.module.mambacore.api.dao.impl;
 
+import org.openmrs.module.dbevent.DbEventSource;
+import org.openmrs.module.dbevent.DbEventSourceConfig;
+import org.openmrs.module.dbevent.EventContext;
 import org.openmrs.module.mambacore.api.dao.FlattenDatabaseDao;
 import org.openmrs.module.mambacore.db.ConnectionPoolManager;
+import org.openmrs.module.mambacore.db.debezium.MyEventConsumer;
 import org.openmrs.module.mambacore.util.MambaETLProperties;
 import org.openmrs.module.mambacore.util.StringReplacerUtil;
 import org.slf4j.Logger;
@@ -17,6 +21,7 @@
 import java.sql.Connection;
 import java.sql.PreparedStatement;
 import java.sql.SQLException;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.stream.Collectors;
@@ -29,13 +34,33 @@ public class JdbcFlattenDatabaseDao implements FlattenDatabaseDao {
     private static final String MYSQL_COMMENT_REGEX = "--.*(?=\\n)";
     private static final String DELIMITER = "~-~-";
 
+    //private DebeziumListener debeziumListener = new DebeziumListener();
+
+    /**
+     * Deploy MambaETL stored procedures
+     */
     @Override
     public void deployMambaEtl() {
 
+        log.info("Deploying MambaETL...");
         MambaETLProperties props = MambaETLProperties.getInstance();
-        log.info("Deploying MambaETL, scheduled @interval: " + props.getInterval() + " seconds...");
         executeSqlScript(props);
-        log.info("Done deploying MambaETL...");
+        log.info("MambaETL deployed (with interval: " + props.getInterval() + "s )...");
+    }
+
+    /**
+     * Stream in database changes using Debezium
+     */
+    @Override
+    public void streamInDatabaseChanges() {
+        //debeziumListener.startListening();
+        EventContext ctx = new EventContext();
+        DbEventSourceConfig config = new DbEventSourceConfig(100002, "mamba-debezium", ctx);
+        config.configureTablesToInclude(Arrays.asList("obs", "patient", "encounter", "encounter_type", "location"));
+        DbEventSource eventSource = new DbEventSource(config);
+        MyEventConsumer consumer = new MyEventConsumer();
+        eventSource.setEventConsumer(consumer);
+        eventSource.start();
     }
 
     private void executeSqlScript(MambaETLProperties props) {

api/src/main/java/org/openmrs/module/mambacore/api/impl/FlattenDatabaseServiceImpl.java

@@ -16,7 +16,7 @@
 import org.slf4j.LoggerFactory;
 import org.springframework.transaction.annotation.Transactional;
 
-import javax.annotation.PreDestroy;
+//import javax.annotation.PreDestroy;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
@@ -38,6 +38,7 @@ public void setupEtl() {
         executorService.submit(() -> {
             try {
                 dao.deployMambaEtl();
+                dao.streamInDatabaseChanges();
             } catch (Exception e) {
                 log.error("Error deploying Mamba ETL", e);
             }
@@ -46,7 +47,7 @@ public void setupEtl() {
 
 
     @Override
-    @PreDestroy
+    //@PreDestroy
     public void shutdownEtlThread() {
         executorService.shutdown();
         try {

api/src/main/java/org/openmrs/module/mambacore/debezium/DbChangeConsumer.java

@@ -0,0 +1,67 @@
+/**
+ * This Source Code Form is subject to the terms of the Mozilla Public License,
+ * v. 2.0. If a copy of the MPL was not distributed with this file, You can
+ * obtain one at http://mozilla.org/MPL/2.0/. OpenMRS is also distributed under
+ * the terms of the Healthcare Disclaimer located at http://openmrs.org/license.
+ * <p>
+ * Copyright (C) OpenMRS Inc. OpenMRS is a registered trademark and the OpenMRS
+ * graphic logo is a trademark of OpenMRS Inc.
+ */
+package org.openmrs.module.mambacore.debezium;
+
+import io.debezium.engine.ChangeEvent;
+import org.apache.kafka.connect.source.SourceRecord;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.function.Consumer;
+
+/**
+ * Implementation of a Debezium ChangeEvent consumer, which abstracts the Debezium API behind a DbCrudEvent
+ * and ensures that the registered DbCrudEvent EventConsumer is successfully processed before moving onto the next
+ * record, with a configurable retryInterval upon failure.
+ **/
+public class DbChangeConsumer implements Consumer<ChangeEvent<SourceRecord, SourceRecord>> {
+
+    private static final Logger logger = LoggerFactory.getLogger(DbChangeConsumer.class);
+
+    private final EventConsumer eventConsumer;
+    private final DbChangeToEventMapper eventMapper;
+    private boolean disabled = false;
+
+    public DbChangeConsumer(EventConsumer eventConsumer, DbChangeToEventMapper eventMapper) {
+        this.eventConsumer = eventConsumer;
+        this.eventMapper = eventMapper;
+    }
+
+    /**
+     * This the primary handler for all Debezium-generated change events.  Per the
+     * <a href="https://debezium.io/documentation/reference/stable/development/engine.html">Debezium Documentation</a>
+     * this function should not throw any exceptions, as these will simply get logged and Debezium will continue onto
+     * the next source record.  So if any exception is caught, this logs the Exception, and retries again after
+     * a configurable retryInterval, until it passes.  This effectively blocks any subsequent processing.
+     *
+     * @param changeEvent the Debeziumn generated event to process
+     */
+    @Override
+    public final void accept(ChangeEvent<SourceRecord, SourceRecord> changeEvent) {
+
+        try {
+            if (disabled) {
+                logger.error("The Debezium consumer has been stopped prior to processing: " + changeEvent);
+                return;
+            }
+
+            DbEvent dbEvent = eventMapper.apply(changeEvent);
+            logger.debug("Notifying listener of the database event: " + dbEvent);
+            eventConsumer.accept(dbEvent);
+        } catch (Throwable t) {
+            logger.error("An error occurred processing change event: " + changeEvent, t);
+            disabled = true;
+        }
+    }
+
+    public void cancel() {
+        this.disabled = true;
+    }
+}

api/src/main/java/org/openmrs/module/mambacore/debezium/DbChangeService.java

@@ -0,0 +1,19 @@
+/**
+ * This Source Code Form is subject to the terms of the Mozilla Public License,
+ * v. 2.0. If a copy of the MPL was not distributed with this file, You can
+ * obtain one at http://mozilla.org/MPL/2.0/. OpenMRS is also distributed under
+ * the terms of the Healthcare Disclaimer located at http://openmrs.org/license.
+ * <p>
+ * Copyright (C) OpenMRS Inc. OpenMRS is a registered trademark and the OpenMRS
+ * graphic logo is a trademark of OpenMRS Inc.
+ */
+package org.openmrs.module.mambacore.debezium;
+
+public interface DbChangeService {
+
+    void start();
+
+    void stop();
+
+    void reset();
+}

api/src/main/java/org/openmrs/module/mambacore/debezium/DbChangeServiceImpl.java

@@ -0,0 +1,100 @@
+/**
+ * This Source Code Form is subject to the terms of the Mozilla Public License,
+ * v. 2.0. If a copy of the MPL was not distributed with this file, You can
+ * obtain one at http://mozilla.org/MPL/2.0/. OpenMRS is also distributed under
+ * the terms of the Healthcare Disclaimer located at http://openmrs.org/license.
+ * <p>
+ * Copyright (C) OpenMRS Inc. OpenMRS is a registered trademark and the OpenMRS
+ * graphic logo is a trademark of OpenMRS Inc.
+ */
+package org.openmrs.module.mambacore.debezium;
+
+import io.debezium.config.Configuration;
+import io.debezium.embedded.Connect;
+import io.debezium.engine.ChangeEvent;
+import io.debezium.engine.DebeziumEngine;
+import io.debezium.engine.spi.OffsetCommitPolicy;
+import org.apache.kafka.connect.source.SourceRecord;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+public class DbChangeServiceImpl implements DbChangeService {
+
+    private static final Logger logger = LoggerFactory.getLogger(DbChangeServiceImpl.class);
+
+    private DbChangeConsumer consumer;
+
+    private Configuration debeziumConfig;
+
+    private RocksDBOffsetBackingStore offsetBackingStore;
+
+    private DebeziumEngine<ChangeEvent<SourceRecord, SourceRecord>> engine;
+
+    private ExecutorService executor;
+
+    @Override
+    public void start() {
+
+        if (engine != null) {
+            logger.warn("Debezium engine is already running.");
+            return;
+        }
+
+        engine = DebeziumEngine.create(Connect.class)
+                .using(debeziumConfig.asProperties())
+                .notifying(consumer)
+                .using(OffsetCommitPolicy.always())
+                .build();
+
+        executor = Executors.newSingleThreadExecutor();
+        executor.execute(engine);
+        Runtime.getRuntime().addShutdownHook(new Thread(this::stop));
+
+        logger.info("Debezium engine started.");
+    }
+
+    @Override
+    public void stop() {
+
+        if (engine == null) {
+            logger.warn("Debezium engine is not running.");
+            return;
+        }
+
+        try {
+            engine.close();
+            executor.shutdown();
+            consumer.cancel();
+            if (offsetBackingStore != null) {
+                offsetBackingStore.stop();
+            }
+
+            try {
+                while (!this.executor.awaitTermination(5L, TimeUnit.SECONDS)) {
+                    logger.info("Waiting 5 seconds for the Debezium engine to shut down...");
+                }
+            } catch (InterruptedException ie) {
+                logger.error("Interrupted while waiting for termination", ie);
+                Thread.currentThread().interrupt();
+            }
+
+        } catch (Exception e) {
+            logger.error("Error stopping Debezium engine", e);
+            executor.shutdownNow();
+        } finally {
+            engine = null;
+            executor = null;
+        }
+    }
+
+    @Override
+    public void reset() {
+        stop();
+        start();
+        logger.info("Debezium engine restarted.");
+    }
+}