Skip to content

Commit

Permalink
add summary for trend ave
Browse files Browse the repository at this point in the history
  • Loading branch information
DawidNiezgodka committed Jan 23, 2024
1 parent 1a8cda1 commit 7e557c9
Show file tree
Hide file tree
Showing 3 changed files with 190 additions and 42 deletions.
116 changes: 95 additions & 21 deletions dist/index.js

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

113 changes: 93 additions & 20 deletions src/comment.js
Original file line number Diff line number Diff line change
Expand Up @@ -272,20 +272,6 @@ module.exports.createBodyForComparisonWithTrendDetMovAverage = function(evaluati
)
lines.push('|-|-|-|-|-|-|')

/**
* return module.exports.createEvaluationObject({
* "evaluation_method": "trend_detection_moving_ave",
* "metric_names": metricNames,
* "metric_units": metricUnits,
* "is": percentageIncreases,
* "should_be": should_be,
* "result": evaluationResults,
* "reference_benchmarks": {
* "previous": previousBenchmarkDataArray,
* "current": currentBenchmarkData
* }
* });
*/

const evaluationResults = evaluationResult.results.result
const evaluationParameters = evaluationResult.evalParameters
Expand All @@ -304,9 +290,6 @@ module.exports.createBodyForComparisonWithTrendDetMovAverage = function(evaluati
const shouldBe = evaluationParameters.shouldBe[i];
const ratio = evaluationParameters.is[i];

//const numberOfConsideredBuilds = evaluationConfiguration.numberOfConsideredBuilds[i];


let line

// Max.Jump | Was | No builds | Res
Expand Down Expand Up @@ -892,7 +875,8 @@ module.exports.summaryForMethodNotSupported = function (evaluationResult, linkTo
//////////
/// Helpers
//////////
module.exports.addSummary = function (evaluationMethod, headers, rows, summaryMessage, linkToGraph, eventName) {
module.exports.addSummary = function (evaluationMethod, headers, rows, summaryMessage, linkToGraph, eventName,
isMovingAve = false, movingAveWindowSize = null) {

const methodSpecificDescription = module.exports.getEvaluationMethodSpecificDescriptionOfEvalMethod(evaluationMethod);
const methodDescriptionFullText = `<b>Method description:</b> ${methodSpecificDescription}`;
Expand All @@ -906,6 +890,11 @@ module.exports.addSummary = function (evaluationMethod, headers, rows, summaryMe
core.summary.addRaw(scheduledEventExtraInfo)
.addSeparator();
}
if (isMovingAve) {
const movingAveExtraInfo = `The chosen moving average window size is ${movingAveWindowSize}.`;
core.summary.addRaw(movingAveExtraInfo)
.addSeparator();
}
core.summary
.addHeading(`The chosen evaluation method: ${evaluationMethod}`, 4)
.addRaw(methodDescriptionFullText)
Expand Down Expand Up @@ -1016,6 +1005,90 @@ module.exports.createWorkflowSummaryForJumpDetection = function (evaluationResul
completeConfig.eventName);
}

module.exports.createWorkflowSummaryForTrendDetAve = function (evaluationResult, completeConfig) {
const currentBenchmark = evaluationResult.referenceBenchmarks.current;
const previousBenchmark = evaluationResult.referenceBenchmarks.previous;

const currentCommitId = completeConfig.eventName === 'schedule' ? currentBenchmark.commitInfo.id : currentBenchmark.commitInfo.id.substring(0, 7);
const previousCommitId = previousBenchmark.commitInfo.eventName === 'schedule' ? previousBenchmark.commitInfo.id : previousBenchmark.commitInfo.id.substring(0, 7);

const headers = [
{
data: 'Metric',
header: true,
},
{
data: `Current: "${currentCommitId}"`,
header: true,
},


{
data: 'Jump',
header: true,
},
{
data: 'Max. change [%]',
header: true,
},
{
data: 'Result',
header: true,
}

];

const rows = [];
const evaluationResults = evaluationResult.results.result
const evaluationParameters = evaluationResult.evalParameters
const evaluationConfiguration = completeConfig.evaluationConfig
for (let i = 0; i < evaluationResults.length; i++) {
const resultStatus = evaluationResults[i];
const metricName = evaluationParameters.metricNames[i];
const metricUnit = evaluationParameters.metricUnits[i];

const currValue = currentBenchmark.simpleMetricResults[i].value;
const currPlusUnit = currValue + ' ' + metricUnit;
const shouldBe = evaluationParameters.shouldBe[i];
const ratio = evaluationParameters.is[i];

let line



let graphicalRepresentationOfRes;
if (resultStatus === 'failed' || resultStatus === 'passed') {
graphicalRepresentationOfRes = resultStatus === 'passed' ? '🟢' : '🔴'
} else {
graphicalRepresentationOfRes= '🔘';
}

rows.push([
{
data: metricName,
},
{
data: currPlusUnit,
},
{
data: ratio,
},
{
data: shouldBe,
},
{
data: graphicalRepresentationOfRes
},

])
}
const movingAveWindowSize = completeConfig.evaluationConfig.movingAveWindowSize;
let summaryMessage = module.exports.createSummaryMessage(evaluationResult);
const evaluationMethod = evaluationResult.evalParameters.evaluationMethod;
module.exports.addSummary(evaluationMethod, headers, rows, summaryMessage, completeConfig.linkToTemplatedGhPageWithResults,
completeConfig.eventName, true, movingAveWindowSize);
}



module.exports.getEvaluationMethodSpecificDescriptionOfEvalMethod = function (evaluationMethod) {
Expand All @@ -1029,9 +1102,9 @@ module.exports.getEvaluationMethodSpecificDescriptionOfEvalMethod = function (ev
case 'threshold_range':
return "The method compares the current benchmark in relation to a range of a given values (given by lower and upper bounds)."
case 'jump_detection':
return ""
return "The strategy checks if the difference between the current and previous value does not exceed a given threshold"
case 'trend_detection_moving_ave':
return ""
return "The procedure checks if the current value does not exceed the average of a particular number of last measurements more than a given threshold"
case 'trend_detection_deltas':
return "The method tries to identify software performance degradation by comparing current performance against three benchmarks:" +
" the immediate previous run, a run closest to one week ago, and the last stable release." +
Expand Down
3 changes: 2 additions & 1 deletion src/main.js
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@ const { createCurrBench} = require('./bench')

const { createComment, createWorkflowSummaryForCompWithPrev, createWorkflowSummaryThreshold,
summaryForMethodNotSupported, createWorkflowSummaryForThresholdRange,
createWorkflowSummaryForTrendDetDeltas, createWorkflowSummaryForJumpDetection} = require('./comment')
createWorkflowSummaryForTrendDetDeltas, createWorkflowSummaryForJumpDetection,
createWorkflowSummaryForTrendDetAve} = require('./comment')

const {
addCompleteBenchmarkToFile,
Expand Down

1 comment on commit 7e557c9

@DawidNiezgodka
Copy link
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Benchmark results

Benchmark group: MongoDB Benchmark

The chosen evaluation method is trend_detection_moving_ave.
For each metric, the procedure checks if the current value does not exceed the average
of a particular number of last measurements more than a given threshold

Execution time: 0m 11s

Parametrization:

  • storageEngine: N/A
  • logVerbosity: N/A
  • bindIp: 0.0.0.0
  • port: 27017

Other Info: YCSB Parameters: workload=workloada, recordcount=200000, threads=32,

Results

Metric Curr: 7e557c9 Max.Jump Was No builds Res
OVERALL Throughput 16747.613465081227 ops/sec 10 75.08 5 🔴
[READ], AverageLatency(us) 1825.4436406295827 us 10 0.81 5 🟢
[READ], MaxLatency(us) 87807 us 10 25.52 5 🔴
[UPDATE], AverageLatency(us) 1871.9681377399565 us 10 1.00 5 🟢
[UPDATE], MaxLatency(us) 151423 us 10 17.72 5 🔴

Benchmark failed

The chosen failing condition was any.
At least one metric didn't pass the tests.

Please sign in to comment.