From 281dc18766207d457c1a2b80a064a9e179384af2 Mon Sep 17 00:00:00 2001
From: William Welling <wwelling@tamu.edu>
Date: Wed, 1 Jan 2025 16:04:47 -0600
Subject: [PATCH] Reduce logging and add timing

---
 .../scholars/discovery/etl/load/SolrIndexLoader.java     | 3 +--
 .../tamu/scholars/discovery/etl/service/EtlService.java  | 9 ++++++---
 .../tamu/scholars/discovery/factory/index/SolrIndex.java | 4 ----
 3 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/src/main/java/edu/tamu/scholars/discovery/etl/load/SolrIndexLoader.java b/src/main/java/edu/tamu/scholars/discovery/etl/load/SolrIndexLoader.java
index 324ebf14..2fe771b3 100644
--- a/src/main/java/edu/tamu/scholars/discovery/etl/load/SolrIndexLoader.java
+++ b/src/main/java/edu/tamu/scholars/discovery/etl/load/SolrIndexLoader.java
@@ -76,13 +76,12 @@ public void destroy() {
 
     @Override
     public void load(Collection<SolrInputDocument> documents) {
-        log.info("Loading {} {} documents", documents.size(), this.data.getName());
         this.index.update(documents);
+        log.info("{} {} documents loaded", documents.size(), this.data.getName());
     }
 
     @Override
     public void load(SolrInputDocument document) {
-        log.debug("Loading {} document", this.data.getName());
         this.index.update(document);
     }
 
diff --git a/src/main/java/edu/tamu/scholars/discovery/etl/service/EtlService.java b/src/main/java/edu/tamu/scholars/discovery/etl/service/EtlService.java
index 0b789258..5375a406 100644
--- a/src/main/java/edu/tamu/scholars/discovery/etl/service/EtlService.java
+++ b/src/main/java/edu/tamu/scholars/discovery/etl/service/EtlService.java
@@ -4,6 +4,7 @@
 import static edu.tamu.scholars.discovery.etl.EtlCacheUtility.VALUES_CACHE;
 import static java.lang.String.format;
 
+import java.time.Duration;
 import java.time.Instant;
 import java.util.Collection;
 import java.util.List;
@@ -50,11 +51,12 @@ public EtlService(
 
     @Override
     public void onApplicationEvent(ContextRefreshedEvent event) {
-        Instant startTime = Instant.now().plusMillis(10000);
-        threadPoolTaskScheduler.schedule(this::process, startTime);
+        threadPoolTaskScheduler.schedule(this::process, Instant.now());
     }
 
     private <I, O> void process() {
+        final Instant start = Instant.now();
+
         List<CompletableFuture<EtlContext<I, O>>> futures = dataRepo.findAll()
                 .stream()
                 .<EtlContext<I, O>>map(this::init)
@@ -68,7 +70,8 @@ private <I, O> void process() {
                         .map(CompletableFuture::join)
                         .toList())
                 .thenAccept(contexts -> {
-                    log.info("All ETL processes completed");
+                    log.info("All ETL processes finished. {} seconds",
+                            Duration.between(start, Instant.now()).toMillis() / 1000.0);
                     contexts.stream().forEach(this::destroy);
                     PROPERTY_CACHE.clear();
                     VALUES_CACHE.clear();
diff --git a/src/main/java/edu/tamu/scholars/discovery/factory/index/SolrIndex.java b/src/main/java/edu/tamu/scholars/discovery/factory/index/SolrIndex.java
index 03688325..92b9d936 100644
--- a/src/main/java/edu/tamu/scholars/discovery/factory/index/SolrIndex.java
+++ b/src/main/java/edu/tamu/scholars/discovery/factory/index/SolrIndex.java
@@ -84,10 +84,8 @@ public JsonNode schema(JsonNode schema) {
     @Override
     public void update(Collection<SolrInputDocument> documents) {
         try {
-            log.info("Updating {} documents to Solr", documents.size());
             this.solrClient.add(documents);
             this.solrClient.commit();
-            log.info("Successfully updated {} documents to Solr", documents.size());
         } catch (RemoteSolrException | SolrServerException | IOException e) {
             log.warn("Error updating Solr documents", e);
             log.info("Attempting batch documents individually");
@@ -109,10 +107,8 @@ public void update(SolrInputDocument document) {
         }
 
         try {
-            // log.info("Updating document to Solr");
             this.solrClient.add(document);
             this.solrClient.commit();
-            // log.info("Successfully updated document to Solr");
         } catch (RemoteSolrException | SolrServerException | IOException e) {
 
             System.out.println("\n\n" + document + "\n\n");