Skip to content

Commit

Permalink
DHFPROD-10074: Stabilize MJS tests in nightly regressions
Browse files Browse the repository at this point in the history
  • Loading branch information
Ryan Dew authored and MarkLogic Builder committed Apr 5, 2023
1 parent f21419b commit 7b4ca19
Show file tree
Hide file tree
Showing 9 changed files with 33 additions and 133 deletions.
87 changes: 4 additions & 83 deletions Jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1077,19 +1077,19 @@ pipeline{
agent {label 'dhfLinuxAgent'}
steps{timeout(time: 3, unit: 'HOURS'){
catchError(buildResult: 'SUCCESS', catchInterruptions: true, stageResult: 'FAILURE') {
singleNodeTestOnLinux('Release','9.0-11')
fullCycleSingleNodeTestOnLinux('Release', '9.0-11')
singleNodeTestOnLinux('Release','10.0-9.5')
fullCycleSingleNodeTestOnLinux('Release', '10.0-9.5')
}}}
post{
success {
println("End-End Tests Completed")
sendMail Email,'<h3>Tests Passed on Released 9.0 ML Server Single Node </h3><h4><a href=${RUN_DISPLAY_URL}>Check the Pipeline View</a></h4><h4> <a href=${BUILD_URL}/console> Check Console Output Here</a></h4>',false,'$BRANCH_NAME branch | Linux RH7 | ML-9.0-11 | Single Node | Passed'
sendMail Email,'<h3>Tests Passed on Released 10.0 ML Server Single Node </h3><h4><a href=${RUN_DISPLAY_URL}>Check the Pipeline View</a></h4><h4> <a href=${BUILD_URL}/console> Check Console Output Here</a></h4>',false,'$BRANCH_NAME branch | Linux RH7 | ML-10.0-9.5 | Single Node | Passed'
}
unstable {
println("End-End Tests Failed")
sh 'mkdir -p MLLogs;cp -r /var/opt/MarkLogic/Logs/* $WORKSPACE/MLLogs/'
archiveArtifacts artifacts: 'MLLogs/**/*'
sendMail Email,'<h3>Some Tests Failed on Released 9.0 ML Server Single Node </h3><h4><a href=${JENKINS_URL}/blue/organizations/jenkins/Datahub_CI/detail/$JOB_BASE_NAME/$BUILD_ID/tests><font color=red>Check the Test Report</font></a></h4><h4><a href=${RUN_DISPLAY_URL}>Check the Pipeline View</a></h4><h4> <a href=${BUILD_URL}/console> Check Console Output Here</a></h4><h4>Please create bugs for the failed regressions and fix them</h4>',false,'$BRANCH_NAME branch | Linux RH7 | ML-9.0-11 | Single Node | Failed'
sendMail Email,'<h3>Some Tests Failed on Released 10.0 ML Server Single Node </h3><h4><a href=${JENKINS_URL}/blue/organizations/jenkins/Datahub_CI/detail/$JOB_BASE_NAME/$BUILD_ID/tests><font color=red>Check the Test Report</font></a></h4><h4><a href=${RUN_DISPLAY_URL}>Check the Pipeline View</a></h4><h4> <a href=${BUILD_URL}/console> Check Console Output Here</a></h4><h4>Please create bugs for the failed regressions and fix them</h4>',false,'$BRANCH_NAME branch | Linux RH7 | ML-10.0-9.5 | Single Node | Failed'
}}
}

Expand Down Expand Up @@ -1138,45 +1138,6 @@ pipeline{
}
}
}
stage('rh7_cluster_9.0-Nightly'){
agent { label 'dhfLinuxAgent'}
steps{
timeout(time: 4, unit: 'HOURS'){
catchError(buildResult: 'SUCCESS', catchInterruptions: true, stageResult: 'FAILURE'){dhflinuxTests("9.0","Latest")}
}}
post{
always{
sh 'rm -rf $WORKSPACE/xdmp'
}
success {
println("rh7_cluster_9.0-Nightly Completed")
sendMail Email,'<h3>Tests Passed on Nigtly 9.0 ML Server Cluster </h3><h4><a href=${RUN_DISPLAY_URL}>Check the Pipeline View</a></h4><h4> <a href=${BUILD_URL}/console> Check Console Output Here</a></h4>',false,'$BRANCH_NAME branch | Linux RH7 | ML-9.0-Nightly | Cluster | Passed'
}
unstable {
println("rh7_cluster_9.0-Nightly Failed")
sendMail Email,'<h3>Some Tests Failed on Nightly 9.0 ML Server Cluster </h3><h4><a href=${JENKINS_URL}/blue/organizations/jenkins/Datahub_CI/detail/$JOB_BASE_NAME/$BUILD_ID/tests><font color=red>Check the Test Report</font></a></h4><h4><a href=${RUN_DISPLAY_URL}>Check the Pipeline View</a></h4><h4> <a href=${BUILD_URL}/console> Check Console Output Here</a></h4><h4>Please create bugs for the failed regressions and fix them</h4>',false,'$BRANCH_NAME branch | Linux RH7 | ML-9.0-Nightly | Cluster | Failed'
}
}
}
stage('rh7_cluster_9.0-11'){
agent { label 'dhfLinuxAgent'}
steps{
timeout(time: 4, unit: 'HOURS'){
catchError(buildResult: 'SUCCESS', catchInterruptions: true, stageResult: 'FAILURE'){dhflinuxTests("9.0-11","Release")}
}}
post{
always{
sh 'rm -rf $WORKSPACE/xdmp'
}
success {
println("rh7_cluster_9.0-11 Tests Completed")
sendMail Email,'<h3>Tests Passed on 9.0-11 ML Server Cluster </h3><h4><a href=${RUN_DISPLAY_URL}>Check the Pipeline View</a></h4><h4> <a href=${BUILD_URL}/console> Check Console Output Here</a></h4>',false,'$BRANCH_NAME branch | Linux RH7 | ML-9.0-11 | Cluster | Passed'
}
unstable {
println("rh7_cluster_9.0-11 Tests Failed")
}
}
}
stage('rh7_cluster_10.0-9.5'){
agent { label 'dhfLinuxAgent'}
steps{
Expand Down Expand Up @@ -1340,26 +1301,6 @@ pipeline{
stage('Windows Core Parallel'){
when { expression {return params.regressions} }
parallel{
stage('w10_SN_9.0-Nightly'){
agent { label 'dhfWinagent'}
steps{
timeout(time: 4, unit: 'HOURS'){
catchError(buildResult: 'SUCCESS', catchInterruptions: true, stageResult: 'FAILURE'){dhfWinTests("9.0","Latest")}
}}
post{
always{
bat 'RMDIR /S/Q xdmp'
}
success {
println("w12_SN_9.0-nightly Tests Completed")
sendMail Email,'<h3>Tests Passed on Nigtly 9.0 ML Server on Windows Platform</h3><h4><a href=${RUN_DISPLAY_URL}>Check the Pipeline View</a></h4><h4> <a href=${BUILD_URL}/console> Check Console Output Here</a></h4>',false,'$BRANCH_NAME branch | Windows W2k12 | ML-9.0-Nightly | Single Node | Passed'
}
unstable {
println("w12_SN_9.0-nightly Tests Failed")
sendMail Email,'<h3>Some Tests Failed on Nightly 9.0 ML Server on Windows Platform </h3><h4><a href=${JENKINS_URL}/blue/organizations/jenkins/Datahub_CI/detail/$JOB_BASE_NAME/$BUILD_ID/tests><font color=red>Check the Test Report</font></a></h4><h4><a href=${RUN_DISPLAY_URL}>Check the Pipeline View</a></h4><h4> <a href=${BUILD_URL}/console> Check Console Output Here</a></h4><h4>Please create bugs for the failed regressions and fix them</h4>',false,'$BRANCH_NAME branch | Windows W2k12 | ML-9.0-Nightly | Single Node | Failed'
}
}
}
stage('w10_SN_11.0-Nightly'){
agent { label 'dhfWinagent'}
steps{
Expand All @@ -1380,26 +1321,6 @@ pipeline{
}
}
}
stage('w10_SN_9.0-11'){
agent { label 'dhfWinagent'}
steps{
timeout(time: 4, unit: 'HOURS'){
catchError(buildResult: 'SUCCESS', catchInterruptions: true, stageResult: 'FAILURE'){dhfWinTests("9.0-11","Release")}
}}
post{
always{
bat 'RMDIR /S/Q xdmp'
}
success {
println("w12_SN_9.0-11 Tests Completed")
sendMail Email,'<h3>Tests Passed on Released 9.0 ML Server on Windows Platform</h3><h4><a href=${RUN_DISPLAY_URL}>Check the Pipeline View</a></h4><h4> <a href=${BUILD_URL}/console> Check Console Output Here</a></h4>',false,'$BRANCH_NAME branch | Windows W2k12 | ML-9.0-11 | Single Node | Passed'
}
unstable {
println("w12_SN_9.0-11 Tests Failed")
sendMail Email,'<h3>Some Tests Failed on Released 9.0 ML Server on Windows Platform </h3><h4><a href=${JENKINS_URL}/blue/organizations/jenkins/Datahub_CI/detail/$JOB_BASE_NAME/$BUILD_ID/tests><font color=red>Check the Test Report</font></a></h4><h4><a href=${RUN_DISPLAY_URL}>Check the Pipeline View</a></h4><h4> <a href=${BUILD_URL}/console> Check Console Output Here</a></h4><h4>Please create bugs for the failed regressions and fix them</h4>',false,'$BRANCH_NAME branch | Windows W2k12 | ML-9.0-11 | Single Node | Failed'
}
}
}
stage('w12_cluster_10.0-9.5'){
agent { label 'dhfWinCluster'}
steps{
Expand Down
2 changes: 1 addition & 1 deletion installer-for-dhs/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ dependencies {
implementation "com.fasterxml.jackson.core:jackson-databind:2.14.1"
implementation "com.beust:jcommander:1.81"
implementation 'commons-io:commons-io:2.11.0'
implementation "org.slf4j:slf4j-api:1.7.36"
implementation "org.slf4j:slf4j-api:2.0.7"

testImplementation(testFixtures(project(":marklogic-data-hub")))
}
Expand Down
2 changes: 1 addition & 1 deletion marklogic-data-hub-api/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ dependencies {
api "com.marklogic:ml-app-deployer:4.4.0"

implementation "org.apache.commons:commons-lang3:3.12.0"
implementation "org.slf4j:slf4j-api:1.7.36"
implementation "org.slf4j:slf4j-api:2.0.7"
}

task sourcesJar(type: Jar, dependsOn: classes) {
Expand Down
5 changes: 4 additions & 1 deletion marklogic-data-hub-central/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,10 @@ dependencies {
because 'gradle automatically ups the version to 2.12.4 but Palamida scan still thinks that version 2.6.7 ' +
'is being used so setting the version forcibly here to avoid confusing Palamida'
}
implementation('org.apache.logging.log4j:log4j-api:2.17.0') {
implementation('org.apache.logging.log4j:log4j-api:2.17.2') {
because 'CVE-2021-44228, CVE-2021-45046, CVE-2021-45105: Log4j vulnerable to remote code execution and other critical security vulnerabilities'
}
implementation('org.apache.logging.log4j:log4j-core:2.17.2') {
because 'CVE-2021-44228, CVE-2021-45046, CVE-2021-45105: Log4j vulnerable to remote code execution and other critical security vulnerabilities'
}
implementation('org.apache.tomcat.embed:tomcat-embed-el:10.0.16') {
Expand Down
2 changes: 1 addition & 1 deletion marklogic-data-hub-client-jar/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ dependencies {
implementation project(":marklogic-data-hub")
implementation "com.beust:jcommander:1.81"
implementation "org.apache.commons:commons-lang3:3.12.0"
implementation "org.slf4j:slf4j-api:1.7.36"
implementation "org.slf4j:slf4j-api:2.0.7"

testImplementation(testFixtures(project(":marklogic-data-hub")))
}
Expand Down
2 changes: 1 addition & 1 deletion marklogic-data-hub-junit5/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ snyk {

dependencies {
api "org.junit.jupiter:junit-jupiter:5.7.2"
api "org.slf4j:slf4j-api:1.7.36"
api "org.slf4j:slf4j-api:2.0.7"
api "org.springframework:spring-test:5.3.24"
api project(":marklogic-data-hub")

Expand Down
2 changes: 1 addition & 1 deletion marklogic-data-hub-test/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ dependencies {

// Because one of the main features of the test helper classes in this project is access to a Logger object,
// this is an API dependency.
api "org.slf4j:slf4j-api:1.7.36"
api "org.slf4j:slf4j-api:2.0.7"

// This is an API dependency due to the test methods that return a class that depends on a JDOM2 class
api "org.jdom:jdom2:2.0.6.1"
Expand Down
7 changes: 5 additions & 2 deletions marklogic-data-hub/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ dependencies {
// Required for CSV support in ingestion steps
implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-csv:2.14.1"

implementation "org.slf4j:slf4j-api:1.7.36"
implementation "org.slf4j:slf4j-api:2.0.7"
implementation "org.jdom:jdom2:2.0.6.1"

// Required for some code that accesses OkHttp APIs directly; intended to match the version of
Expand Down Expand Up @@ -95,7 +95,10 @@ dependencies {
testImplementation 'org.hamcrest:hamcrest-junit:2.0.0.0'
testImplementation 'org.easymock:easymock:3.4'
testImplementation 'org.apache.logging.log4j:log4j-api:2.17.2'
testImplementation 'org.slf4j:log4j-over-slf4j:1.7.36'
testImplementation 'org.apache.logging.log4j:log4j-core:2.17.2'
testImplementation 'org.apache.logging.log4j:log4j-slf4j-impl:2.17.2'
testImplementation 'org.slf4j:log4j-over-slf4j:2.0.7'
testImplementation 'org.slf4j:slf4j-simple:2.0.7'
testImplementation("com.marklogic:mlcp:10.0.9.5") {
exclude group: 'commons-modeler', module: 'commons-modeler'
exclude group: 'ch.qos.logback', module: 'logback-classic'
Expand Down
57 changes: 15 additions & 42 deletions marklogic-data-hub/src/test/java/com/marklogic/hub/mlcp/MlcpRunner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,45 +119,17 @@ public void run() {
}

private String buildLoggerconfig() {
return "<Configuration status=\"WARN\">\n" +
" <Appenders>\n" +
" <!-- Console appender configuration -->\n" +
" <Console name=\"Console\" target=\"SYSTEM_OUT\">\n" +
" <PatternLayout pattern=\"%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n\"/>\n" +
" </Console>\n" +
" </Appenders>\n" +
" <Loggers>\n" +
" <!-- Root logger configuration -->\n" +
" <Root level=\"INFO\">\n" +
" <AppenderRef ref=\"Console\"/>\n" +
" </Root>\n" +
" <!-- To enable debug for mapreduce -->\n" +
" <!--\n" +
" <Logger name=\"com.marklogic.mapreduce\" level=\"DEBUG\" additivity=\"false\">\n" +
" <AppenderRef ref=\"Console\"/>\n" +
" </Logger>\n" +
" -->\n" +
" <!-- To enable debug for contentpump -->\n" +
" <!--\n" +
" <Logger name=\"com.marklogic.contentpump\" level=\"DEBUG\" additivity=\"false\">\n" +
" <AppenderRef ref=\"Console\"/>\n" +
" </Logger>\n" +
" -->\n" +
" <!-- To enable debug for tree -->\n" +
" <!--\n" +
" <Logger name=\"com.marklogic.tree\" level=\"TRACE\" additivity=\"false\">\n" +
" <AppenderRef ref=\"Console\"/>\n" +
" </Logger>\n" +
" -->\n" +
" <!-- To supress not native warn on Mac and Solaris -->\n" +
" <Logger name=\"org.apache.hadoop.util.NativeCodeLoader\" level=\"ERROR\" additivity=\"false\">\n" +
" <AppenderRef ref=\"Console\"/>\n" +
" </Logger>\n" +
" <Logger name=\"org.apache.hadoop.ipc.Client\" level=\"ERROR\" additivity=\"false\">\n" +
" <AppenderRef ref=\"Console\"/>\n" +
" </Logger>\n" +
" </Loggers>\n" +
"</Configuration>\n";
return "appender.stdout.name = stdout\n" +
"appender.stdout.type = Console\n" +
"logger.app = INFO, stdout, file\n" +
"\n" +
"# Direct log messages to stdout\n" +
"appender.console.type = Console\n" +
"appender.console.name = STDOUT\n" +
"appender.console.layout.type = PatternLayout\n" +
"appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n\n" +
"appender.console.filter.threshold.type = ThresholdFilter\n" +
"appender.console.filter.threshold.level = info";
}

private MlcpBean makeMlcpBean() throws Exception {
Expand Down Expand Up @@ -207,13 +179,14 @@ private void buildCommand(MlcpBean bean) throws IOException {
File.separator + "bin" +
File.separator + "java";
String classpath = System.getProperty("java.class.path");
File loggerFile = File.createTempFile("mlcp-", "-logger.xml");
Path tempDirectory = Files.createTempDirectory("mlcp-");
File loggerFile = tempDirectory.resolve("log4j2.properties").toFile();
FileUtils.writeStringToFile(loggerFile, buildLoggerconfig());
Map<String, String> environment = new HashMap<>();
Map<String, String> environment = new HashMap<>();
classpath = classpath + ";" + loggerFile.toURI();
environment.put("CLASSPATH", classpath);
this.withEnvironment(environment);
args.add(javaBin);
args.add("-Dlog4j2.configurationFile=" + loggerFile.toURI());
if (classpath.endsWith(".war")) {
args.add("-jar");
args.add("mlcp");
Expand Down

0 comments on commit 7b4ca19

Please sign in to comment.