Skip to content

Commit

Permalink
Add in data source name and options to error validation details, writ…
Browse files Browse the repository at this point in the history 
…e the main css and svg resources directly into report folder
  • Loading branch information
@pflooky
pflooky committed Jul 9, 2024
1 parent c6fb039 commit 786648a
Show file tree
Hide file tree
Showing 4 changed files with 199 additions and 39 deletions.
38 changes: 5 additions & 33 deletions ...io/github/datacatering/datacaterer/core/generator/result/DataGenerationResultWriter.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import com.fasterxml.jackson.annotation.JsonInclude.Include
import io.github.datacatering.datacaterer.api.model.Constants.{DEFAULT_GENERATED_REPORTS_FOLDER_PATH, SPECIFIC_DATA_SOURCE_OPTIONS}
import io.github.datacatering.datacaterer.api.model.{DataCatererConfiguration, Field, Plan, Step, Task}
import io.github.datacatering.datacaterer.core.listener.SparkRecordListener
import io.github.datacatering.datacaterer.core.model.Constants.{REPORT_DATA_SOURCES_HTML, REPORT_FIELDS_HTML, REPORT_HOME_HTML, REPORT_VALIDATIONS_HTML}
import io.github.datacatering.datacaterer.core.model.Constants.{REPORT_DATA_CATERING_SVG, REPORT_DATA_SOURCES_HTML, REPORT_FIELDS_HTML, REPORT_HOME_HTML, REPORT_MAIN_CSS, REPORT_RESULT_JSON, REPORT_TASK_HTML, REPORT_VALIDATIONS_HTML}
import io.github.datacatering.datacaterer.core.model.{DataSourceResult, DataSourceResultSummary, StepResultSummary, TaskResultSummary, ValidationConfigResult}
import io.github.datacatering.datacaterer.core.plan.PostPlanProcessor
import io.github.datacatering.datacaterer.core.util.FileUtil.writeStringToFile
Expand Down Expand Up @@ -50,47 +50,19 @@ class DataGenerationResultWriter(val dataCatererConfiguration: DataCatererConfig
try {
fileWriter(REPORT_HOME_HTML, htmlWriter.index(plan, stepSummary, taskSummary, dataSourceSummary,
validationResults, dataCatererConfiguration.flagsConfig, sparkRecordListener))
fileWriter("tasks.html", htmlWriter.taskDetails(taskSummary))
fileWriter(REPORT_TASK_HTML, htmlWriter.taskDetails(taskSummary))
fileWriter(REPORT_FIELDS_HTML, htmlWriter.stepDetails(stepSummary))
fileWriter(REPORT_DATA_SOURCES_HTML, htmlWriter.dataSourceDetails(stepSummary.flatMap(_.dataSourceResults)))
fileWriter(REPORT_VALIDATIONS_HTML, htmlWriter.validations(validationResults, validationConfig))
writeStringToFile(fileSystem, s"$reportFolder/results.json", resultsAsJson(generationResult, validationResults))

copyHtmlResources(fileSystem, reportFolder)
writeStringToFile(fileSystem, s"$reportFolder/$REPORT_RESULT_JSON", resultsAsJson(generationResult, validationResults))
writeStringToFile(fileSystem, s"$reportFolder/$REPORT_DATA_CATERING_SVG", htmlWriter.dataCateringSvg)
writeStringToFile(fileSystem, s"$reportFolder/$REPORT_MAIN_CSS", htmlWriter.mainCss)
} catch {
case ex: Exception =>
LOGGER.error("Failed to write data generation summary to HTML files", ex)
}
}

private def copyHtmlResources(fileSystem: FileSystem, folder: String): Unit = {
val resources = List("main.css", "data_catering_transparent.svg")
resources.foreach(resource => {
val defaultResourcePath = new Path(s"file:///$DEFAULT_GENERATED_REPORTS_FOLDER_PATH/$resource")
val localResourcePath = new Path(s"file://app/src/main/resources/report/$resource")
val tryLocalUri = Try(new Path(getClass.getResource(s"/report/$resource").toURI))
val resourcePath = tryLocalUri match {
case Failure(_) =>
Try(defaultResourcePath.toUri) match {
case Failure(_) => localResourcePath
case Success(_) => defaultResourcePath
}
case Success(value) =>
Try(value.getName) match {
case Failure(_) =>
Try(defaultResourcePath.toUri) match {
case Failure(_) => localResourcePath
case Success(_) => defaultResourcePath
}
case Success(name) =>
if (name.startsWith("jar:")) defaultResourcePath else value
}
}
val destination = s"file:///$folder/$resource"
fileSystem.copyFromLocalFile(resourcePath, new Path(destination))
})
}

private def writeToFile(fileSystem: FileSystem, folderPath: String)(fileName: String, content: Node): Unit = {
writeStringToFile(fileSystem, s"$folderPath/$fileName", content.toString())
}
Expand Down
179 changes: 179 additions & 0 deletions ...ain/scala/io/github/datacatering/datacaterer/core/generator/result/ResultHtmlWriter.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -739,6 +739,183 @@ class ResultHtmlWriter {
</script>
}

def mainCss: String = {
""".box-iframe {
| float: left;
| margin-right: 10px;
|}
|
|body {
| margin: 0;
|}
|
|.top-banner {
| height: fit-content;
| background-color: #ff6e42;
| padding: 0 .2rem;
| display: flex;
|}
|
|.top-banner span {
| color: #f2f2f2;
| font-size: 17px;
| padding: 5px 6px;
| display: flex;
| align-items: center;
|}
|
|.logo {
| padding: 5px;
| height: 45px;
| width: auto;
| display: flex;
| align-items: center;
| justify-content: center;
|}
|
|.logo:hover {
| background-color: #ff9100;
| color: black;
|}
|
|.top-banner img {
| height: 35px;
| width: auto;
| display: flex;
| justify-content: center;
| vertical-align: middle;
|}
|
|.topnav {
| overflow: hidden;
| background-color: #ff6e42;
|}
|
|.topnav a {
| float: left;
| color: #f2f2f2;
| text-align: center;
| padding: 8px 10px;
| text-decoration: none;
| font-size: 17px;
|}
|
|.topnav a:hover {
| background-color: #ff9100;
| color: black;
|}
|
|.topnav a.active {
| color: black;
|}
|
|table {
| overflow: hidden;
| transition: max-height 0.2s ease-out;
|}
|
|table.codegrid {
| font-family: monospace;
| font-size: 12px;
| width: auto !important;
|}
|
|table.statementlist {
| width: auto !important;
| font-size: 13px;
|}
|
|table.codegrid td {
| padding: 0 !important;
| border: 0 !important
|}
|
|table td.linenumber {
| width: 40px !important;
|}
|
|td {
| white-space: normal
|}
|
|.table thead th {
| position: sticky;
| top: 0;
| z-index: 1;
|}
|
|table, tr, td, th {
| border-collapse: collapse;
|}
|
|.table-collapsible {
| max-height: 0;
| overflow: hidden;
| transition: max-height 0.2s ease-out;
|}
|
|.collapsible {
| background-color: lightgray;
| color: black;
| cursor: pointer;
| width: 100%;
| border: none;
| text-align: left;
| outline: none;
|}
|
|.collapsible:after {
| content: "\02795"; /* Unicode character for "plus" sign (+) */
| color: white;
| float: right;
|}
|
|.active:after {
| content: "\2796"; /* Unicode character for "minus" sign (-) */
|}
|
|.outer-container {
| display: flex;
| flex-direction: column;
| height: 100vh;
|}
|
|.top-container {
| height: 50%;
| overflow: auto;
| resize: vertical;
|}
|
|.bottom-container {
| flex: 1;
| min-height: 0;
| height: 50%;
| overflow: auto;
| resize: vertical;
|}
|
|.slider {
| text-align: center;
| background-color: #dee2e6;
| cursor: row-resize;
| user-select: none;
|}
|
|.selected-row {
| background-color: #ff6e42 !important;
|}
|
|.progress {
| white-space: normal;
| background-color: #d9534f;
|}
|
|.progress-bar {
| color: black;
|}
|""".stripMargin
}

def plugins: NodeBuffer = {
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/jquery.tablesorter/2.20.1/css/theme.default.min.css" type="text/css"/>
Expand All @@ -749,4 +926,6 @@ class ResultHtmlWriter {
<link rel="stylesheet" href="main.css" type="text/css"/>
<link rel="icon" href="data_catering_transparent.svg"/>
}

def dataCateringSvg: String = "<svg xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\" width=\"1080\" zoomAndPan=\"magnify\" viewBox=\"0 0 810 809.999993\" height=\"1080\" preserveAspectRatio=\"xMidYMid meet\" version=\"1.0\"><defs><g/><clipPath id=\"d7af0a96c1\"><path d=\"M 43.5 10 L 767.203125 10 L 767.203125 800 L 43.5 800 Z M 43.5 10 \" clip-rule=\"nonzero\"/></clipPath><clipPath id=\"0d6c067e67\"><path d=\"M 43.5 281.675781 L 43.5 528.097656 C 43.5 580.964844 71.75 629.835938 117.597656 656.269531 L 331.273438 779.472656 C 377.121094 805.90625 433.621094 805.90625 479.464844 779.472656 L 693.144531 656.269531 C 738.988281 629.835938 767.238281 580.964844 767.238281 528.097656 L 767.238281 281.675781 C 767.238281 228.808594 738.988281 179.9375 693.144531 153.503906 L 479.464844 30.300781 C 433.621094 3.867188 377.121094 3.867188 331.273438 30.300781 L 117.597656 153.503906 C 71.75 179.957031 43.5 228.808594 43.5 281.675781 Z M 43.5 281.675781 \" clip-rule=\"nonzero\"/></clipPath></defs><g clip-path=\"url(#d7af0a96c1)\"><g clip-path=\"url(#0d6c067e67)\"><path fill=\"#36699f\" d=\"M 43.5 803.773438 L 43.5 6.226562 L 767.203125 6.226562 L 767.203125 803.773438 Z M 43.5 803.773438 \" fill-opacity=\"1\" fill-rule=\"nonzero\"/></g></g><g fill=\"#ff00ff\" fill-opacity=\"1\"><g transform=\"translate(206.24592, 640.079332)\"><g><path d=\"M 197.644531 0.652344 C 345.714844 0 407.03125 -73.710938 407.03125 -232.214844 C 407.03125 -391.375 345.714844 -452.039062 197.644531 -452.039062 L 105.671875 -452.039062 C 69.796875 -452.039062 50.226562 -431.816406 50.226562 -393.984375 C 50.226562 -196.339844 49.574219 -113.5 48.921875 -54.140625 C 48.921875 -18.917969 68.492188 0 104.367188 0 Z M 171.554688 -90.015625 C 170.902344 -112.195312 170.902344 -136.328125 170.902344 -159.8125 C 170.902344 -271.355469 170.902344 -317.015625 171.554688 -362.023438 L 195.035156 -362.023438 C 265.484375 -362.023438 286.355469 -326.144531 286.355469 -226.34375 C 286.355469 -138.9375 262.875 -90.015625 195.035156 -90.015625 Z M 171.554688 -90.015625 \"/></g></g></g><g fill=\"#00ffff\" fill-opacity=\"1\"><g transform=\"translate(181.784409, 640.079332)\"><g><path d=\"M 197.644531 0.652344 C 345.714844 0 407.03125 -73.710938 407.03125 -232.214844 C 407.03125 -391.375 345.714844 -452.039062 197.644531 -452.039062 L 105.671875 -452.039062 C 69.796875 -452.039062 50.226562 -431.816406 50.226562 -393.984375 C 50.226562 -196.339844 49.574219 -113.5 48.921875 -54.140625 C 48.921875 -18.917969 68.492188 0 104.367188 0 Z M 171.554688 -90.015625 C 170.902344 -112.195312 170.902344 -136.328125 170.902344 -159.8125 C 170.902344 -271.355469 170.902344 -317.015625 171.554688 -362.023438 L 195.035156 -362.023438 C 265.484375 -362.023438 286.355469 -326.144531 286.355469 -226.34375 C 286.355469 -138.9375 262.875 -90.015625 195.035156 -90.015625 Z M 171.554688 -90.015625 \"/></g></g></g><g fill=\"#fbcccc\" fill-opacity=\"1\"><g transform=\"translate(194.015166, 640.079332)\"><g><path d=\"M 197.644531 0.652344 C 345.714844 0 407.03125 -73.710938 407.03125 -232.214844 C 407.03125 -391.375 345.714844 -452.039062 197.644531 -452.039062 L 105.671875 -452.039062 C 69.796875 -452.039062 50.226562 -431.816406 50.226562 -393.984375 C 50.226562 -196.339844 49.574219 -113.5 48.921875 -54.140625 C 48.921875 -18.917969 68.492188 0 104.367188 0 Z M 171.554688 -90.015625 C 170.902344 -112.195312 170.902344 -136.328125 170.902344 -159.8125 C 170.902344 -271.355469 170.902344 -317.015625 171.554688 -362.023438 L 195.035156 -362.023438 C 265.484375 -362.023438 286.355469 -326.144531 286.355469 -226.34375 C 286.355469 -138.9375 262.875 -90.015625 195.035156 -90.015625 Z M 171.554688 -90.015625 \"/></g></g></g></svg>"
}
4 changes: 4 additions & 0 deletions app/src/main/scala/io/github/datacatering/datacaterer/core/model/Constants.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,9 +54,13 @@ object Constants {
lazy val COUNT_NUM_RECORDS = "numRecords"

//report
lazy val REPORT_DATA_CATERING_SVG = "data_catering_transparent.svg"
lazy val REPORT_DATA_SOURCES_HTML = "data-sources.html"
lazy val REPORT_FIELDS_HTML = "steps.html"
lazy val REPORT_HOME_HTML = "index.html"
lazy val REPORT_MAIN_CSS = "main.css"
lazy val REPORT_RESULT_JSON = "results.json"
lazy val REPORT_TASK_HTML = "tasks.html"
lazy val REPORT_VALIDATIONS_HTML = "validations.html"

//connection group type
Expand Down
17 changes: 11 additions & 6 deletions app/src/main/scala/io/github/datacatering/datacaterer/core/model/ValidationModels.scala
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package io.github.datacatering.datacaterer.core.model

import io.github.datacatering.datacaterer.api.model.{ExpressionValidation, Validation}
import io.github.datacatering.datacaterer.core.util.ConfigUtil.cleanseOptions
import io.github.datacatering.datacaterer.core.util.ResultWriterUtil.getSuccessSymbol
import org.apache.spark.sql.DataFrame
import org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema
Expand Down Expand Up @@ -29,15 +30,19 @@ case class ValidationConfigResult(
}

def jsonSummary(numErrorSamples: Int): Map[String, Any] = {
val validationRes = dataSourceValidationResults.flatMap(_.validationResults)
val validationRes = dataSourceValidationResults.flatMap(dsv =>
dsv.validationResults.map(v => (dsv.dataSourceName, dsv.options, v))
)
if (validationRes.nonEmpty) {
val (numSuccess, successRate, isSuccess) = baseSummary(validationRes)
val errorMap = validationRes.filter(!_.isSuccess).map(res => {
val validationDetails = res.validation.toOptions.map(v => (v.head, v.last)).toMap
val (numSuccess, successRate, isSuccess) = baseSummary(validationRes.map(_._3))
val errorMap = validationRes.filter(vr => !vr._3.isSuccess).map(res => {
val validationDetails = res._3.validation.toOptions.map(v => (v.head, v.last)).toMap
Map(
"dataSourceName" -> res._1,
"options" -> cleanseOptions(res._2),
"validation" -> validationDetails,
"numErrors" -> res.numErrors,
"sampleErrorValues" -> getErrorSamplesAsMap(numErrorSamples, res)
"numErrors" -> res._3.numErrors,
"sampleErrorValues" -> getErrorSamplesAsMap(numErrorSamples, res._3)
)
})
val baseValidationMap = Map(
Expand Down

0 comments on commit 786648a

Please sign in to comment.