Skip to content

Commit

Permalink
jump det sum
Browse files Browse the repository at this point in the history
  • Loading branch information
DawidNiezgodka committed Jan 22, 2024
1 parent d486fe7 commit daaba46
Show file tree
Hide file tree
Showing 3 changed files with 206 additions and 2 deletions.
104 changes: 103 additions & 1 deletion dist/index.js

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

100 changes: 100 additions & 0 deletions src/comment.js
Original file line number Diff line number Diff line change
Expand Up @@ -843,6 +843,106 @@ module.exports.addSummary = function (evaluationMethod, headers, rows, summaryMe
.write();
}

module.exports.createWorkflowSummaryForJumpDetection = function (evaluationResult, completeConfig) {
const currentBenchmark = evaluationResult.referenceBenchmarks.current;
const previousBenchmark = evaluationResult.referenceBenchmarks.previous;

const currentCommitId = completeConfig.eventName === 'schedule' ? currentBenchmark.commitInfo.id : currentBenchmark.commitInfo.id.substring(0, 7);
const previousCommitId = previousBenchmark.commitInfo.eventName === 'schedule' ? previousBenchmark.commitInfo.id : previousBenchmark.commitInfo.id.substring(0, 7);

const headers = [
{
data: 'Metric',
header: true,
},
{
data: `Current: "${currentCommitId}"`,
header: true,
},
{
data: `Previous: "${previousCommitId}"`,
header: true,
},

{
data: 'Jump',
header: true,
},
{
data: 'Max. change [%]',
header: true,
},
{
data: 'Result',
header: true,
}

];

const rows = [];
const evaluationResults = evaluationResult.results.result
const evaluationParameters = evaluationResult.evalParameters
const evaluationConfiguration = completeConfig.evaluationConfig
for (let i = 0; i < evaluationResults.length; i++) {
const resultStatus = evaluationResults[i];
const metricName = evaluationParameters.metricNames[i];
const metricUnit = evaluationParameters.metricUnits[i];

const currValue = currentBenchmark.simpleMetricResults[i].value;
const prevValue = previousBenchmark.simpleMetricResults[i].value;

const currPlusUnit = currValue + ' ' + metricUnit;
const prevPlusUnit = prevValue + ' ' + metricUnit;

const shouldBe = evaluationParameters.shouldBe[i];
const ratio = evaluationParameters.is[i];



const metricValues = evaluationParameters.metricToDifferentBenchValues.get(metricName);
if (!metricValues) {
continue;
}

let currBenchValue = metricValues?.current ?? 'N/A';
let prevBenchValue = metricValues?.previous ?? 'N/A';

let graphicalRepresentationOfRes;
if (resultStatus === 'failed' || resultStatus === 'passed') {
graphicalRepresentationOfRes = resultStatus === 'passed' ? '🟢' : '🔴'
} else {
graphicalRepresentationOfRes= '🔘';
}

rows.push([
{
data: metricName,
},
{
data: currPlusUnit,
},
{
data: prevPlusUnit,
},
{
data: ratio,
},
{
data: shouldBe,
},
{
data: graphicalRepresentationOfRes
},

])
}
let summaryMessage = module.exports.createSummaryMessage(evaluationResult);
const evaluationMethod = evaluationResult.evalParameters.evaluationMethod;
module.exports.addSummary(evaluationMethod, headers, rows, summaryMessage, completeConfig.linkToTemplatedGhPageWithResults,
completeConfig.eventName);
}



module.exports.getEvaluationMethodSpecificDescriptionOfEvalMethod = function (evaluationMethod) {
switch (evaluationMethod) {
Expand Down
4 changes: 3 additions & 1 deletion src/main.js
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ const { createCurrBench} = require('./bench')

const { createComment, createWorkflowSummaryForCompWithPrev, createWorkflowSummaryThreshold,
summaryForMethodNotSupported, createWorkflowSummaryForThresholdRange,
createWorkflowSummaryForTrendDetDeltas} = require('./comment')
createWorkflowSummaryForTrendDetDeltas, createWorkflowSummaryForJumpDetection} = require('./comment')

const {
addCompleteBenchmarkToFile,
Expand Down Expand Up @@ -118,6 +118,8 @@ async function run() {
createWorkflowSummaryForThresholdRange(evaluationResult, completeConfig)
} else if (evaluationConfig.evaluationMethod === 'trend_detection_deltas') {
createWorkflowSummaryForTrendDetDeltas(evaluationResult, completeConfig);
} else if (evaluationConfig.evaluationMethod === 'jump_detection') {
createWorkflowSummaryForJumpDetection(evaluationResult, completeConfig);
}

else {
Expand Down

1 comment on commit daaba46

@DawidNiezgodka
Copy link
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Benchmark results

Benchmark group: MongoDB Benchmark

The chosen evaluation method is jump_detection.
For each metric, there is the following condition:
The current value should not change more than X% (Max. ch in the table below) from the value measured for the previous benchmark.

Current Benchmark Previous Benchmark
Execution time: 0m 11s Execution time: 0m 11s
Parametrization: Parametrization:
- storageEngine: N/A - storageEngine: N/A
- logVerbosity: N/A - logVerbosity: N/A
- bindIp: 0.0.0.0 - bindIp: 0.0.0.0
- port: 27017 - port: 27017
Other Info: YCSB Parameters: workload=workloada, recordcount=200000, threads=32, Other Info: YCSB Parameters: workload=workloada, recordcount=200000, threads=32,

Results

Metric Curr: daaba46 Prev: 8f60099 Max. Jump Was Res
OVERALL Throughput 16747.613465081227 ops/sec 67213.42 ops/sec 10 75.08 🔴
[READ], AverageLatency(us) 1825.4436406295827 us 1810.7 us 10 0.81 🟢
[READ], MaxLatency(us) 87807 us 117887 us 10 25.52 🔴
[UPDATE], AverageLatency(us) 1871.9681377399565 us 1853.37 us 10 1.00 🟢
[UPDATE], MaxLatency(us) 151423 us 128631 us 10 17.72 🔴

Benchmark failed

The chosen failing condition was any.
At least one metric didn't pass the tests.

Please sign in to comment.