PHOENIX-6857 Enable code coverage reporting to SonarQube in Phoenix-C…

…onnectors
apache · Apr 25, 2023 · 2f998ea · 2f998ea
1 parent 229c155
commit 2f998ea
Show file tree

Hide file tree

Showing 20 changed files with 219 additions and 5 deletions.
diff --git a/dev/code-coverage/README.md b/dev/code-coverage/README.md
@@ -0,0 +1,41 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+     http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Code analysis
+
+The `run-coverage.sh` script runs maven with the **coverage** profile
+which generates the test coverage data for the java classes.
+If the required parameters are given it also runs the sonar code analysis
+and uploads the results to the given SonarQube Server.
+
+## Running code analysis
+
+After running the script the reports generated by the JaCoCo code coverage
+library can be found in the modules under the `/target/site/jacoco/` folder.
+
+Here is how you can generate the code coverage report:
+
+```./dev/code-coverage/run-coverage.sh```
+## Publishing coverage results to SonarQube
+The required parameters for publishing the results to SonarQube are:
+- host URL,
+- login credentials,
+- project key
+
+The project name is an optional parameter.
+
+Here is an example command for running and publishing the coverage data:
+`sh dev/code-coverage/run-coverage.sh -l ProjectCredentials -u https://exampleserver.com -k Project_Key -n Project_Name`
diff --git a/dev/code-coverage/run-coverage.sh b/dev/code-coverage/run-coverage.sh
@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+usage() {
+  echo
+  echo "Options:"
+  echo "  -h     Display help"
+  echo "  -u     SonarQube Host URL"
+  echo "  -l     SonarQube Login Credentials"
+  echo "  -k     SonarQube Project Key"
+  echo "  -n     SonarQube Project Name"
+  echo "  -m     Additional Maven options"
+  echo "  -t     Number of threads (example: 1 or 2C)."
+  echo
+  echo "Important:"
+  echo "  The required parameters for publishing the coverage results to SonarQube are:"
+  echo "    - Host URL"
+  echo "    - Login Credentials"
+  echo "    - Project Key"
+  echo
+}
+
+execute() {
+  SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
+  MAIN_POM="${SCRIPT_DIR}/../../pom.xml"
+
+  echo "Run tests"
+  mvn -B -e -f "${MAIN_POM}" clean install -Pcoverage -fn -Dmaven.javadoc.skip=true
+
+  echo "Run sonar analysis"
+  # If the required parameters are given, the code coverage results are uploaded to the SonarQube Server
+  if [ -n "$SONAR_LOGIN" ] && [ -n "$SONAR_PROJECT_KEY" ] && [ -n "$SONAR_URL" ]; then
+    mvn -B -e -f "${MAIN_POM}" -Pcoverage sonar:sonar -Dsonar.host.url="$SONAR_URL" \
+      -Dsonar.login="$SONAR_LOGIN" -Dsonar.projectKey="$SONAR_PROJECT_KEY" -Dsonar.projectName="$SONAR_PROJECT_NAME"
+  fi
+}
+
+while getopts ":u:l:k:n:h" option; do
+  case $option in
+  u) SONAR_URL=${OPTARG:-} ;;
+  l) SONAR_LOGIN=${OPTARG:-} ;;
+  k) SONAR_PROJECT_KEY=${OPTARG:-} ;;
+  n) SONAR_PROJECT_NAME=${OPTARG:-} ;;
+  h) # Display usage
+    usage
+    exit
+    ;;
+  \?) # Invalid option
+    echo "Error: Invalid option"
+    exit
+    ;;
+  esac
+done
+
+# Start code analysis
+execute
diff --git a/phoenix-flume-base/phoenix4-flume/pom.xml b/phoenix-flume-base/phoenix4-flume/pom.xml
@@ -32,6 +32,10 @@
   <artifactId>phoenix4-flume</artifactId>
   <name>Phoenix Flume Connector for Phoenix 4</name>
 
+  <properties>
+    <sonar.skip>true</sonar.skip>
+  </properties>
+
   <dependencies>
     <dependency>
       <groupId>org.apache.phoenix</groupId>

diff --git a/phoenix-flume-base/phoenix5-flume/pom.xml b/phoenix-flume-base/phoenix5-flume/pom.xml
@@ -40,6 +40,7 @@
     <hadoop.version>${hadoop-three.version}</hadoop.version>
     <jdk.version>1.8</jdk.version>
     <phoenix.main.version>5</phoenix.main.version>
+    <sonar.skip>true</sonar.skip>
   </properties>
 
   <dependencies>

diff --git a/phoenix-flume-base/pom.xml b/phoenix-flume-base/pom.xml
@@ -39,6 +39,9 @@
 
   <properties>
     <top.dir>${project.basedir}/..</top.dir>
+    <sonar.sources>src/main/</sonar.sources>
+    <sonar.test.sources>src/it/</sonar.test.sources>
+    <sonar.exclusions>src/**/it/**/*</sonar.exclusions>
   </properties>
 
   <dependencies>

diff --git a/phoenix-hive-base/phoenix4-hive-shaded/pom.xml b/phoenix-hive-base/phoenix4-hive-shaded/pom.xml
@@ -35,6 +35,7 @@
   <properties>
     <hive.version>${hive2.version}</hive.version>
     <hive-storage.version>${hive2-storage.version}</hive-storage.version>
+    <sonar.skip>true</sonar.skip>
   </properties>
 
   <dependencies>

diff --git a/phoenix-hive-base/phoenix4-hive/pom.xml b/phoenix-hive-base/phoenix4-hive/pom.xml
@@ -35,6 +35,8 @@
   <properties>
     <hive.version>${hive2.version}</hive.version>
     <hive-storage.version>${hive2-storage.version}</hive-storage.version>
+    <sonar.tests></sonar.tests>
+    <sonar.test.exclusions>**/src/**/*</sonar.test.exclusions>
   </properties>
 
   <dependencies>

diff --git a/phoenix-hive-base/phoenix5-hive-shaded/pom.xml b/phoenix-hive-base/phoenix5-hive-shaded/pom.xml
@@ -42,6 +42,7 @@
     <hive.version>${hive3.version}</hive.version>
     <jdk.version>1.8</jdk.version>
     <phoenix.main.version>5</phoenix.main.version>
+    <sonar.skip>true</sonar.skip>
   </properties>
 
   <dependencies>

diff --git a/phoenix-hive-base/phoenix5-hive/pom.xml b/phoenix-hive-base/phoenix5-hive/pom.xml
@@ -43,6 +43,8 @@
     <jdk.version>1.8</jdk.version>
     <phoenix.main.version>5</phoenix.main.version>
     <calcite.version>${hive5.calcite.version}</calcite.version>
+    <sonar.tests></sonar.tests>
+    <sonar.test.exclusions>**/src/**/*</sonar.test.exclusions>
   </properties>
 
   <dependencies>

diff --git a/phoenix-hive-base/pom.xml b/phoenix-hive-base/pom.xml
@@ -50,6 +50,9 @@
     <hive4.calcite.version>1.10.0</hive4.calcite.version>
     <hive5.calcite.version>1.16.0</hive5.calcite.version>
     <calcite.version>${hive4.calcite.version}</calcite.version>
+    <sonar.sources>src/main/</sonar.sources>
+    <sonar.test.sources>src/it/,src/test/</sonar.test.sources>
+    <sonar.exclusions>src/**/it/**/*,src/**/test/**/*</sonar.exclusions>
   </properties>
 
   <dependencies>

diff --git a/phoenix-kafka-base/phoenix4-kafka/pom.xml b/phoenix-kafka-base/phoenix4-kafka/pom.xml
@@ -32,6 +32,10 @@
   <artifactId>phoenix4-kafka</artifactId>
   <name>Phoenix Kafka Connector for Phoenix 4</name>
 
+  <properties>
+    <sonar.skip>true</sonar.skip>
+  </properties>
+
   <dependencies>
     <dependency>
       <groupId>org.apache.phoenix</groupId>

diff --git a/phoenix-kafka-base/phoenix5-kafka/pom.xml b/phoenix-kafka-base/phoenix5-kafka/pom.xml
@@ -40,6 +40,7 @@
     <hadoop.version>${hadoop-three.version}</hadoop.version>
     <jdk.version>1.8</jdk.version>
     <phoenix.main.version>5</phoenix.main.version>
+    <sonar.skip>true</sonar.skip>
   </properties>
 
 

diff --git a/phoenix-kafka-base/pom.xml b/phoenix-kafka-base/pom.xml
@@ -40,6 +40,8 @@
 
   <properties>
     <top.dir>${project.basedir}/..</top.dir>
+    <sonar.sources>src/main/</sonar.sources>
+    <sonar.exclusions>src/**/it/**/*</sonar.exclusions>
   </properties>
 
   <dependencies>

diff --git a/phoenix-spark-base/phoenix4-spark-shaded/pom.xml b/phoenix-spark-base/phoenix4-spark-shaded/pom.xml
@@ -32,6 +32,10 @@
   <artifactId>phoenix4-spark-shaded</artifactId>
   <name>Shaded Phoenix Spark Connector for Phoenix 4</name>
 
+  <properties>
+    <sonar.skip>true</sonar.skip>
+  </properties>
+
   <dependencies>
     <dependency>
       <groupId>org.apache.phoenix</groupId>

diff --git a/phoenix-spark-base/phoenix4-spark/pom.xml b/phoenix-spark-base/phoenix4-spark/pom.xml
@@ -32,6 +32,10 @@
   <artifactId>phoenix4-spark</artifactId>
   <name>Phoenix Spark Connector for Phoenix 4</name>
 
+  <properties>
+    <sonar.skip>true</sonar.skip>
+  </properties>
+
   <dependencies>
     <dependency>
       <groupId>org.apache.phoenix</groupId>

diff --git a/phoenix-spark-base/phoenix5-spark-shaded/pom.xml b/phoenix-spark-base/phoenix5-spark-shaded/pom.xml
@@ -40,6 +40,7 @@
     <hadoop.version>${hadoop-three.version}</hadoop.version>
     <jdk.version>1.8</jdk.version>
     <phoenix.main.version>5</phoenix.main.version>
+    <sonar.skip>true</sonar.skip>
   </properties>
 
   <dependencies>

diff --git a/phoenix-spark-base/phoenix5-spark/pom.xml b/phoenix-spark-base/phoenix5-spark/pom.xml
@@ -40,6 +40,7 @@
     <hadoop.version>${hadoop-three.version}</hadoop.version>
     <jdk.version>1.8</jdk.version>
     <phoenix.main.version>5</phoenix.main.version>
+    <sonar.skip>true</sonar.skip>
   </properties>
 
   <dependencies>

diff --git a/phoenix-spark-base/pom.xml b/phoenix-spark-base/pom.xml
@@ -36,6 +36,8 @@
   <properties>
     <top.dir>${project.basedir}/..</top.dir>
     <skip-scala-tests>true</skip-scala-tests>
+    <sonar.sources>src/main/</sonar.sources>
+    <sonar.exclusions>src/**/test/,src/**/it/</sonar.exclusions>
   </properties>
 
   <packaging>pom</packaging>
@@ -591,7 +593,7 @@
                   to https://github.com/junit-team/junit4/issues/1223 -->
                 <parallel>false</parallel>
                 <tagsToExclude>Integration-Test</tagsToExclude>
-                <argLine>-Xmx1536m -XX:ReservedCodeCacheSize=512m</argLine>
+                <argLine>@{jacocoArgLine} -Xmx1536m -XX:ReservedCodeCacheSize=512m</argLine>
               </configuration>
             </execution>
           </executions>

diff --git a/phoenix5-spark3-it/pom.xml b/phoenix5-spark3-it/pom.xml
@@ -209,7 +209,7 @@
                 to https://github.com/junit-team/junit4/issues/1223 -->
               <parallel>false</parallel>
               <tagsToExclude>Integration-Test</tagsToExclude>
-              <argLine>-Xmx1536m -XX:ReservedCodeCacheSize=512m</argLine>
+              <argLine>@{jacocoArgLine} -Xmx1536m -XX:ReservedCodeCacheSize=512m</argLine>
             </configuration>
           </execution>
         </executions>

diff --git a/pom.xml b/pom.xml
@@ -145,6 +145,11 @@
     <phoenix.main.version>4</phoenix.main.version>
 
     <skip-scala-tests>true</skip-scala-tests>
+
+    <!-- Coverage properties -->
+    <jacoco.version>0.8.8</jacoco.version>
+    <sonar.version>3.9.1.2184</sonar.version>
+    <jacocoArgLine />
   </properties>
 
   <build>
@@ -312,13 +317,14 @@
             <shutdown>kill</shutdown>
             <testSourceDirectory>${basedir}/src/it/java</testSourceDirectory>
             <trimStackTrace>false</trimStackTrace>
+            <includes>**/*IT*</includes>
           </configuration>
           <executions>
             <execution>
               <id>ParallelStatsDisabledTest</id>
               <configuration>
                 <reuseForks>true</reuseForks>
-                <argLine>-Xmx3000m -XX:MaxPermSize=256m -Djava.security.egd=file:/dev/./urandom "-Djava.library.path=${hadoop.library.path}${path.separator}${java.library.path}" -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=./target/</argLine>
+                <argLine>@{jacocoArgLine} -Xmx3000m -XX:MaxPermSize=256m -Djava.security.egd=file:/dev/./urandom "-Djava.library.path=${hadoop.library.path}${path.separator}${java.library.path}" -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=./target/</argLine>
                 <groups>org.apache.phoenix.end2end.ParallelStatsDisabledTest</groups>
               </configuration>
               <goals>
@@ -330,7 +336,7 @@
               <id>NeedsOwnMiniClusterTest</id>
               <configuration>
                 <reuseForks>false</reuseForks>
-                <argLine>-enableassertions -Xmx3000m -XX:MaxPermSize=256m -Djava.security.egd=file:/dev/./urandom "-Djava.library.path=${hadoop.library.path}${path.separator}${java.library.path}" -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=./target/</argLine>
+                <argLine>@{jacocoArgLine} -enableassertions -Xmx3000m -XX:MaxPermSize=256m -Djava.security.egd=file:/dev/./urandom "-Djava.library.path=${hadoop.library.path}${path.separator}${java.library.path}" -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=./target/</argLine>
                 <groups>org.apache.phoenix.end2end.NeedsOwnMiniClusterTest</groups>
               </configuration>
               <goals>
@@ -414,11 +420,12 @@
         <configuration>
           <forkCount>${numForkedUT}</forkCount>
           <reuseForks>true</reuseForks>
-          <argLine>-enableassertions -Xmx2250m -XX:MaxPermSize=128m
+          <argLine>@{jacocoArgLine} -enableassertions -Xmx2250m -XX:MaxPermSize=128m
             -Djava.security.egd=file:/dev/./urandom "-Djava.library.path=${hadoop.library.path}${path.separator}${java.library.path}" -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=./target/</argLine>
           <redirectTestOutputToFile>${test.output.tofile}</redirectTestOutputToFile>
           <shutdown>kill</shutdown>
           <trimStackTrace>false</trimStackTrace>
+          <includes>**/*Test*</includes>
         </configuration>
       </plugin>
       <!-- All projects create a test jar -->
@@ -975,6 +982,65 @@
         </plugins>
       </build>
     </profile>
+    <profile>
+      <id>coverage</id>
+      <activation>
+        <activeByDefault>false</activeByDefault>
+      </activation>
+      <properties>
+        <sonar.coverage.exclusions>
+          **/examples/**/*,**/phoenix-connectors-phoenix4-compat/**/*,**/phoenix-connectors-phoenix5-compat/**/*,
+          **/phoenix4-connectors-assembly/**/*,**/phoenix5-connectors-assembly/**/*,**/scala/**/*
+        </sonar.coverage.exclusions>
+        <sonar.java.binaries>**/target/classes</sonar.java.binaries>
+        <sonar.surefire.reportsPath>${project.build.directory}/surefire-reports</sonar.surefire.reportsPath>
+        <sonar.projectBaseDir>.</sonar.projectBaseDir>
+        <sonar.coverage.jacoco.xmlReportPaths>
+          ${sonar.projectBaseDir}/phoenix-hive-base/phoenix4-hive/target/site/jacoco/jacoco.xml,
+          ${sonar.projectBaseDir}/phoenix-hive-base/phoenix5-hive/target/site/jacoco/jacoco.xml,
+          ${sonar.projectBaseDir}/phoenix-flume-base/phoenix4-flume/target/site/jacoco/jacoco.xml,
+          ${sonar.projectBaseDir}/phoenix-flume-base/phoenix5-flume/target/site/jacoco/jacoco.xml,
+          ${sonar.projectBaseDir}/phoenix-kafka-base/phoenix4-kafka/target/site/jacoco/jacoco.xml,
+          ${sonar.projectBaseDir}/phoenix-kafka-base/phoenix5-kafka/target/site/jacoco/jacoco.xml,
+          ${sonar.projectBaseDir}/phoenix-spark-base/phoenix4-spark/target/site/jacoco/jacoco.xml,
+          ${sonar.projectBaseDir}/phoenix-spark-base/phoenix5-spark/target/site/jacoco/jacoco.xml,
+          ${sonar.projectBaseDir}/phoenix5-spark3/target/site/jacoco/jacoco.xml
+        </sonar.coverage.jacoco.xmlReportPaths>
+      </properties>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.jacoco</groupId>
+            <artifactId>jacoco-maven-plugin</artifactId>
+            <version>${jacoco.version}</version>
+            <executions>
+              <execution>
+                <id>prepare-agent</id>
+                <phase>initialize</phase>
+                <goals>
+                  <goal>prepare-agent</goal>
+                </goals>
+                <configuration>
+                  <propertyName>jacocoArgLine</propertyName>
+                  <append>true</append>
+                </configuration>
+              </execution>
+              <execution>
+                <id>report</id>
+                <goals>
+                  <goal>report</goal>
+                </goals>
+              </execution>
+            </executions>
+          </plugin>
+          <plugin>
+            <groupId>org.sonarsource.scanner.maven</groupId>
+            <artifactId>sonar-maven-plugin</artifactId>
+            <version>${sonar.version}</version>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
   </profiles>
 
   <reporting>