Skip to content

Commit

Permalink
[Metrics UI] Add overrides to Snapshot API to support alert previews (#…
Browse files Browse the repository at this point in the history
…68125)

* [Metrics UI] Add overrides to Snapshot API to support alert previews

* Renaming tests from infra to metrics_ui; renaming waffle to snapshot; adding tests for overrides

* removing limit and afterKey; removing extra interval

* Setting the minimum value for lookbackSize
  • Loading branch information
simianhacker authored Jun 3, 2020
1 parent 0279b18 commit 0b8e0c0
Show file tree
Hide file tree
Showing 23 changed files with 144 additions and 31 deletions.
20 changes: 15 additions & 5 deletions x-pack/plugins/infra/common/http_api/snapshot_api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

import * as rt from 'io-ts';
import { SnapshotMetricTypeRT, ItemTypeRT } from '../inventory_models/types';
import { metricsExplorerSeriesRT } from './metrics_explorer';

export const SnapshotNodePathRT = rt.intersection([
rt.type({
Expand All @@ -21,6 +22,7 @@ const SnapshotNodeMetricOptionalRT = rt.partial({
value: rt.union([rt.number, rt.null]),
avg: rt.union([rt.number, rt.null]),
max: rt.union([rt.number, rt.null]),
timeseries: metricsExplorerSeriesRT,
});

const SnapshotNodeMetricRequiredRT = rt.type({
Expand All @@ -41,11 +43,18 @@ export const SnapshotNodeResponseRT = rt.type({
interval: rt.string,
});

export const InfraTimerangeInputRT = rt.type({
interval: rt.string,
to: rt.number,
from: rt.number,
});
export const InfraTimerangeInputRT = rt.intersection([
rt.type({
interval: rt.string,
to: rt.number,
from: rt.number,
}),
rt.partial({
lookbackSize: rt.number,
ignoreLookback: rt.boolean,
forceInterval: rt.boolean,
}),
]);

export const SnapshotGroupByRT = rt.array(
rt.partial({
Expand Down Expand Up @@ -97,6 +106,7 @@ export const SnapshotRequestRT = rt.intersection([
accountId: rt.string,
region: rt.string,
filterQuery: rt.union([rt.string, rt.null]),
includeTimeseries: rt.boolean,
}),
]);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ import {
SnapshotNodeResponseRT,
SnapshotNodeResponse,
SnapshotGroupBy,
SnapshotRequest,
InfraTimerangeInput,
} from '../../../../../common/http_api/snapshot_api';
import {
InventoryItemType,
Expand All @@ -37,10 +39,11 @@ export function useSnapshot(
);
};

const timerange = {
const timerange: InfraTimerangeInput = {
interval: '1m',
to: currentTime,
from: currentTime - 360 * 1000,
lookbackSize: 20,
};

const { error, loading, response, makeRequest } = useHTTPRequest<SnapshotNodeResponse>(
Expand All @@ -55,7 +58,8 @@ export function useSnapshot(
sourceId,
accountId,
region,
}),
includeTimeseries: true,
} as SnapshotRequest),
decodeResponse
);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,30 +12,48 @@ import { SnapshotModel, SnapshotModelMetricAggRT } from '../../../common/invento
import { getDatasetForField } from '../../routes/metrics_explorer/lib/get_dataset_for_field';
import { InfraTimerangeInput } from '../../../common/http_api/snapshot_api';
import { ESSearchClient } from '.';
import { getIntervalInSeconds } from '../../utils/get_interval_in_seconds';

export const createTimeRangeWithInterval = async (
client: ESSearchClient,
options: InfraSnapshotRequestOptions
): Promise<InfraTimerangeInput> => {
const createInterval = async (client: ESSearchClient, options: InfraSnapshotRequestOptions) => {
const { timerange } = options;
if (timerange.forceInterval && timerange.interval) {
return getIntervalInSeconds(timerange.interval);
}
const aggregations = getMetricsAggregations(options);
const modules = await aggregationsToModules(client, aggregations, options);
const interval = Math.max(
return Math.max(
(await calculateMetricInterval(
client,
{
indexPattern: options.sourceConfiguration.metricAlias,
timestampField: options.sourceConfiguration.fields.timestamp,
timerange: { from: options.timerange.from, to: options.timerange.to },
timerange: { from: timerange.from, to: timerange.to },
},
modules,
options.nodeType
)) || 60,
60
);
};

export const createTimeRangeWithInterval = async (
client: ESSearchClient,
options: InfraSnapshotRequestOptions
): Promise<InfraTimerangeInput> => {
const { timerange } = options;
const calculatedInterval = await createInterval(client, options);
if (timerange.ignoreLookback) {
return {
interval: `${calculatedInterval}s`,
from: timerange.from,
to: timerange.to,
};
}
const lookbackSize = Math.max(timerange.lookbackSize || 5, 5);
return {
interval: `${interval}s`,
from: options.timerange.to - interval * 5000, // We need at least 5 buckets worth of data
to: options.timerange.to,
interval: `${calculatedInterval}s`,
from: timerange.to - calculatedInterval * lookbackSize * 1000, // We need at least 5 buckets worth of data
to: timerange.to,
};
};

Expand Down
23 changes: 22 additions & 1 deletion x-pack/plugins/infra/server/lib/snapshot/response_helpers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import { isNumber, last, max, sum, get } from 'lodash';
import moment from 'moment';

import { MetricsExplorerSeries } from '../../../common/http_api/metrics_explorer';
import { getIntervalInSeconds } from '../../utils/get_interval_in_seconds';
import { InfraSnapshotRequestOptions } from './types';
import { findInventoryModel } from '../../../common/inventory_models';
Expand Down Expand Up @@ -127,12 +128,15 @@ export const getNodeMetrics = (
};
}
const lastBucket = findLastFullBucket(nodeBuckets, options);
const result = {
const result: SnapshotNodeMetric = {
name: options.metric.type,
value: getMetricValueFromBucket(options.metric.type, lastBucket),
max: calculateMax(nodeBuckets, options.metric.type),
avg: calculateAvg(nodeBuckets, options.metric.type),
};
if (options.includeTimeseries) {
result.timeseries = getTimeseriesData(nodeBuckets, options.metric.type);
}
return result;
};

Expand Down Expand Up @@ -164,3 +168,20 @@ function calculateMax(buckets: InfraSnapshotMetricsBucket[], type: SnapshotMetri
function calculateAvg(buckets: InfraSnapshotMetricsBucket[], type: SnapshotMetricType) {
return sum(buckets.map((bucket) => getMetricValueFromBucket(type, bucket))) / buckets.length || 0;
}

function getTimeseriesData(
buckets: InfraSnapshotMetricsBucket[],
type: SnapshotMetricType
): MetricsExplorerSeries {
return {
id: type,
columns: [
{ name: 'timestamp', type: 'date' },
{ name: 'metric_0', type: 'number' },
],
rows: buckets.map((bucket) => ({
timestamp: bucket.key as number,
metric_0: getMetricValueFromBucket(type, bucket),
})),
};
}
2 changes: 2 additions & 0 deletions x-pack/plugins/infra/server/routes/snapshot/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ export const initSnapshotRoute = (libs: InfraBackendLibs) => {
timerange,
accountId,
region,
includeTimeseries,
} = pipe(
SnapshotRequestRT.decode(request.body),
fold(throwErrors(Boom.badRequest), identity)
Expand All @@ -57,6 +58,7 @@ export const initSnapshotRoute = (libs: InfraBackendLibs) => {
sourceConfiguration: source.configuration,
metric,
timerange,
includeTimeseries,
};

const searchES = <Hit = {}, Aggregation = undefined>(
Expand Down
2 changes: 1 addition & 1 deletion x-pack/test/api_integration/apis/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ export default function ({ loadTestFile }) {
loadTestFile(require.resolve('./telemetry'));
loadTestFile(require.resolve('./logstash'));
loadTestFile(require.resolve('./kibana'));
loadTestFile(require.resolve('./infra'));
loadTestFile(require.resolve('./metrics_ui'));
loadTestFile(require.resolve('./beats'));
loadTestFile(require.resolve('./console'));
loadTestFile(require.resolve('./management'));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
*/

export default function ({ loadTestFile }) {
describe('InfraOps Endpoints', () => {
describe('MetricsUI Endpoints', () => {
loadTestFile(require.resolve('./metadata'));
loadTestFile(require.resolve('./log_analysis'));
loadTestFile(require.resolve('./log_entries'));
Expand All @@ -15,7 +15,7 @@ export default function ({ loadTestFile }) {
loadTestFile(require.resolve('./log_summary'));
loadTestFile(require.resolve('./metrics'));
loadTestFile(require.resolve('./sources'));
loadTestFile(require.resolve('./waffle'));
loadTestFile(require.resolve('./snapshot'));
loadTestFile(require.resolve('./log_item'));
loadTestFile(require.resolve('./metrics_alerting'));
loadTestFile(require.resolve('./metrics_explorer'));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,25 +10,15 @@ import { first, last } from 'lodash';
import {
InfraSnapshotMetricInput,
InfraNodeType,
InfraTimerangeInput,
InfraSnapshotGroupbyInput,
} from '../../../../plugins/infra/server/graphql/types';
import { FtrProviderContext } from '../../ftr_provider_context';
import {
SnapshotNodeResponse,
SnapshotMetricInput,
SnapshotRequest,
} from '../../../../plugins/infra/common/http_api/snapshot_api';
import { DATES } from './constants';

interface SnapshotRequest {
filterQuery?: string | null;
metric: SnapshotMetricInput;
groupBy: InfraSnapshotGroupbyInput[];
nodeType: InfraNodeType;
sourceId: string;
timerange: InfraTimerangeInput;
}

export default function ({ getService }: FtrProviderContext) {
const esArchiver = getService('esArchiver');
const supertest = getService('supertest');
Expand Down Expand Up @@ -200,6 +190,74 @@ export default function ({ getService }: FtrProviderContext) {
});
});

it('should allow for overrides for interval and ignoring lookback', () => {
const resp = fetchSnapshot({
sourceId: 'default',
timerange: {
to: max,
from: min,
interval: '10s',
forceInterval: true,
ignoreLookback: true,
},
metric: { type: 'cpu' } as InfraSnapshotMetricInput,
nodeType: 'host' as InfraNodeType,
groupBy: [],
includeTimeseries: true,
});
return resp.then((data) => {
const snapshot = data;
expect(snapshot).to.have.property('nodes');
if (snapshot) {
const { nodes } = snapshot;
expect(nodes.length).to.equal(1);
const firstNode = first(nodes);
expect(firstNode).to.have.property('path');
expect(firstNode.path.length).to.equal(1);
expect(first(firstNode.path)).to.have.property('value', 'demo-stack-mysql-01');
expect(first(firstNode.path)).to.have.property('label', 'demo-stack-mysql-01');
expect(firstNode).to.have.property('metric');
expect(firstNode.metric).to.have.property('timeseries');
expect(firstNode.metric.timeseries?.rows.length).to.equal(58);
const rows = firstNode.metric.timeseries?.rows;
const rowInterval = (rows?.[1]?.timestamp || 0) - (rows?.[0]?.timestamp || 0);
expect(rowInterval).to.equal(10000);
}
});
});

it('should allow for overrides for lookback', () => {
const resp = fetchSnapshot({
sourceId: 'default',
timerange: {
to: max,
from: min,
interval: '1m',
lookbackSize: 6,
},
metric: { type: 'cpu' } as InfraSnapshotMetricInput,
nodeType: 'host' as InfraNodeType,
groupBy: [],
includeTimeseries: true,
});
return resp.then((data) => {
const snapshot = data;
expect(snapshot).to.have.property('nodes');
if (snapshot) {
const { nodes } = snapshot;
expect(nodes.length).to.equal(1);
const firstNode = first(nodes);
expect(firstNode).to.have.property('path');
expect(firstNode.path.length).to.equal(1);
expect(first(firstNode.path)).to.have.property('value', 'demo-stack-mysql-01');
expect(first(firstNode.path)).to.have.property('label', 'demo-stack-mysql-01');
expect(firstNode).to.have.property('metric');
expect(firstNode.metric).to.have.property('timeseries');
expect(firstNode.metric.timeseries?.rows.length).to.equal(7);
}
});
});

it('should work with custom metrics', async () => {
const data = await fetchSnapshot({
sourceId: 'default',
Expand Down

0 comments on commit 0b8e0c0

Please sign in to comment.