From 010441694154b1a0fa2eab0f114ebcf1dbde9a57 Mon Sep 17 00:00:00 2001
From: Marek Horst <marek.horst@gmail.com>
Date: Tue, 3 Oct 2023 12:46:48 +0200
Subject: [PATCH] Closes #1426: Run IIS experiments by relying on spark 3.4
 version

WIP.

Removing `provided` scope from the `spark-avro_2.12` dependency until making it part of sharelib342.
Introducing required fixes for `eu/dnetlib/iis/wf/export/actionmanager/relation/citation/default` integration test to let it run relying on spark3:
* setting `spark.extraListeners` and `spark.sql.queryExecutionListeners` explicitly to empty values in order to avoid relying on incompatible, spark2 compliant, cloudera listeners
* setting `spark.shuffle.useOldFetchProtocol=true` to address `2.4 to 3.0 migration guide` requirement regarding protocol for fetching shuffle blocks backward compatibility (and avoiding `IllegalArgumentException: Unexpected message type: <number>` kind of errors)
---
 .../relation/citation/oozie_app/workflow.xml      | 15 +++------------
 pom.xml                                           |  3 +++
 2 files changed, 6 insertions(+), 12 deletions(-)

diff --git a/iis-wf/iis-wf-export-actionmanager/src/main/resources/eu/dnetlib/iis/wf/export/actionmanager/relation/citation/oozie_app/workflow.xml b/iis-wf/iis-wf-export-actionmanager/src/main/resources/eu/dnetlib/iis/wf/export/actionmanager/relation/citation/oozie_app/workflow.xml
index d242d7cf7..64b982ce7 100644
--- a/iis-wf/iis-wf-export-actionmanager/src/main/resources/eu/dnetlib/iis/wf/export/actionmanager/relation/citation/oozie_app/workflow.xml
+++ b/iis-wf/iis-wf-export-actionmanager/src/main/resources/eu/dnetlib/iis/wf/export/actionmanager/relation/citation/oozie_app/workflow.xml
@@ -42,16 +42,6 @@
             <name>oozieActionShareLibForSpark2</name>
             <description>oozie action sharelib for spark 2.*</description>
         </property>
-        <property>
-            <name>spark2ExtraListeners</name>
-            <value>com.cloudera.spark.lineage.NavigatorAppListener</value>
-            <description>spark 2.* extra listeners classname</description>
-        </property>
-        <property>
-            <name>spark2SqlQueryExecutionListeners</name>
-            <value>com.cloudera.spark.lineage.NavigatorQueryListener</value>
-            <description>spark 2.* sql query execution listeners classname</description>
-        </property>
         <property>
             <name>spark2YarnHistoryServerAddress</name>
             <description>spark 2.* yarn history server address</description>
@@ -94,10 +84,11 @@
                 --executor-memory=${sparkExecutorMemory}
                 --executor-cores=${sparkExecutorCores}
                 --driver-memory=${sparkDriverMemory}
-                --conf spark.extraListeners=${spark2ExtraListeners}
-                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.extraListeners=
+                --conf spark.sql.queryExecutionListeners=
                 --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
                 --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+                --conf spark.shuffle.useOldFetchProtocol=true
             </spark-opts>
             <arg>-inputCitationsPath=${input_citations}</arg>
             <arg>-outputRelationPath=${output_root_relations}/${action_set_id_citation_relations}</arg>
diff --git a/pom.xml b/pom.xml
index 311c23cd7..dcb4d8ac8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -387,7 +387,10 @@
                 <groupId>org.apache.spark</groupId>
                 <artifactId>spark-avro_2.12</artifactId>
                 <version>${iis.spark.version}</version>
+                <!-- FIXME not available in spark3 sharelib folder yet, commenting out until finishing the testing phase -->
+                <!--
                 <scope>provided</scope>
+                -->
             </dependency>
             
             <dependency>