Skip to content

Commit

Permalink
[8.10] [ML] API integration tests for model_management endpoints (#…
Browse files Browse the repository at this point in the history
…164668) (#164764)

# Backport

This will backport the following commits from `main` to `8.10`:
- [[ML] API integration tests for `model_management` endpoints
(#164668)](#164668)

<!--- Backport version: 8.9.7 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Dima
Arnautov","email":"[email protected]"},"sourceCommit":{"committedDate":"2023-08-24T17:32:34Z","message":"[ML]
API integration tests for `model_management` endpoints
(#164668)","sha":"b3f036b554600ae5f6ce330c91b4829636267ff5","branchLabelMapping":{"^v8.11.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":[":ml","test_api","release_note:skip","Team:ML","v8.10.0","v8.11.0"],"number":164668,"url":"https://github.com/elastic/kibana/pull/164668","mergeCommit":{"message":"[ML]
API integration tests for `model_management` endpoints
(#164668)","sha":"b3f036b554600ae5f6ce330c91b4829636267ff5"}},"sourceBranch":"main","suggestedTargetBranches":["8.10"],"targetPullRequestStates":[{"branch":"8.10","label":"v8.10.0","labelRegex":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"main","label":"v8.11.0","labelRegex":"^v8.11.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/164668","number":164668,"mergeCommit":{"message":"[ML]
API integration tests for `model_management` endpoints
(#164668)","sha":"b3f036b554600ae5f6ce330c91b4829636267ff5"}}]}]
BACKPORT-->

Co-authored-by: Dima Arnautov <[email protected]>
  • Loading branch information
kibanamachine and darnautov authored Aug 24, 2023
1 parent bef38d4 commit f7aeb6c
Show file tree
Hide file tree
Showing 5 changed files with 230 additions and 0 deletions.
1 change: 1 addition & 0 deletions x-pack/test/api_integration/apis/ml/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -73,5 +73,6 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) {
loadTestFile(require.resolve('./system'));
loadTestFile(require.resolve('./trained_models'));
loadTestFile(require.resolve('./notifications'));
loadTestFile(require.resolve('./model_management'));
});
}
15 changes: 15 additions & 0 deletions x-pack/test/api_integration/apis/ml/model_management/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import { FtrProviderContext } from '../../../ftr_provider_context';

export default function ({ loadTestFile }: FtrProviderContext) {
describe('model management', function () {
loadTestFile(require.resolve('./memory_usage'));
loadTestFile(require.resolve('./nodes_overview'));
});
}
107 changes: 107 additions & 0 deletions x-pack/test/api_integration/apis/ml/model_management/memory_usage.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import expect from '@kbn/expect';
import { Datafeed, Job } from '@kbn/ml-plugin/common/types/anomaly_detection_jobs';
import { FtrProviderContext } from '../../../ftr_provider_context';
import { USER } from '../../../../functional/services/ml/security_common';
import { getCommonRequestHeader } from '../../../../functional/services/ml/common_api';

export default ({ getService }: FtrProviderContext) => {
const supertest = getService('supertestWithoutAuth');
const ml = getService('ml');
const esArchiver = getService('esArchiver');

// @ts-expect-error not full interface
const JOB_CONFIG: Job = {
job_id: `fq_multi_1_ae`,
description:
'mean/min/max(responsetime) partition=airline on farequote dataset with 1h bucket span',
groups: ['farequote', 'automated', 'multi-metric'],
analysis_config: {
bucket_span: '1h',
influencers: ['airline'],
detectors: [
{ function: 'mean', field_name: 'responsetime', partition_field_name: 'airline' },
{ function: 'min', field_name: 'responsetime', partition_field_name: 'airline' },
{ function: 'max', field_name: 'responsetime', partition_field_name: 'airline' },
],
},
data_description: { time_field: '@timestamp' },
analysis_limits: { model_memory_limit: '20mb' },
model_plot_config: { enabled: true },
};

// @ts-expect-error not full interface
const DATAFEED_CONFIG: Datafeed = {
datafeed_id: 'datafeed-fq_multi_1_ae',
indices: ['ft_farequote'],
job_id: 'fq_multi_1_ae',
query: { bool: { must: [{ match_all: {} }] } },
};

async function createMockJobs() {
await ml.api.createAnomalyDetectionJob(JOB_CONFIG);
await ml.api.createDatafeed(DATAFEED_CONFIG);
await ml.api.openAnomalyDetectionJob(JOB_CONFIG.job_id);
await ml.api.startDatafeed(DATAFEED_CONFIG.datafeed_id);
}

describe('GET model_management/memory_usage', () => {
before(async () => {
await ml.testResources.setKibanaTimeZoneToUTC();
await ml.api.createTestTrainedModels('regression', 2);
await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/farequote');
await createMockJobs();
});

after(async () => {
await ml.api.closeAnomalyDetectionJob(JOB_CONFIG.job_id);
await ml.api.cleanMlIndices();
await ml.testResources.cleanMLSavedObjects();
});

it('returns model memory usage', async () => {
const { body, status } = await supertest
.get(`/internal/ml/model_management/memory_usage`)
.auth(USER.ML_POWERUSER, ml.securityCommon.getPasswordForUser(USER.ML_POWERUSER))
.set(getCommonRequestHeader('1'));
ml.api.assertResponseStatusCode(200, status, body);

expect(body[0].id).to.eql('fq_multi_1_ae');
expect(body[0].type).to.eql('anomaly-detector');
expect(body[0].size).to.greaterThan(10000000);
});

it('filters out memory usage response based on the entity type', async () => {
const { body, status } = await supertest
.get(`/internal/ml/model_management/memory_usage`)
.query({ type: 'data-frame-analytics' })
.auth(USER.ML_POWERUSER, ml.securityCommon.getPasswordForUser(USER.ML_POWERUSER))
.set(getCommonRequestHeader('1'));
ml.api.assertResponseStatusCode(200, status, body);

expect(body).to.eql([]);
});

it('returns an error for the user with viewer permissions', async () => {
const { body, status } = await supertest
.get(`/internal/ml/model_management/memory_usage`)
.auth(USER.ML_VIEWER, ml.securityCommon.getPasswordForUser(USER.ML_VIEWER))
.set(getCommonRequestHeader('1'));
ml.api.assertResponseStatusCode(403, status, body);
});

it('returns an error for unauthorized user', async () => {
const { body, status } = await supertest
.get(`/internal/ml/model_management/memory_usage`)
.auth(USER.ML_UNAUTHORIZED, ml.securityCommon.getPasswordForUser(USER.ML_UNAUTHORIZED))
.set(getCommonRequestHeader('1'));
ml.api.assertResponseStatusCode(403, status, body);
});
});
};
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import expect from '@kbn/expect';
import { Datafeed, Job } from '@kbn/ml-plugin/common/types/anomaly_detection_jobs';
import { FtrProviderContext } from '../../../ftr_provider_context';
import { USER } from '../../../../functional/services/ml/security_common';
import { getCommonRequestHeader } from '../../../../functional/services/ml/common_api';

export default ({ getService }: FtrProviderContext) => {
const supertest = getService('supertestWithoutAuth');
const ml = getService('ml');
const esArchiver = getService('esArchiver');

// @ts-expect-error not full interface
const JOB_CONFIG: Job = {
job_id: `fq_multi_1_ae`,
description:
'mean/min/max(responsetime) partition=airline on farequote dataset with 1h bucket span',
groups: ['farequote', 'automated', 'multi-metric'],
analysis_config: {
bucket_span: '1h',
influencers: ['airline'],
detectors: [
{ function: 'mean', field_name: 'responsetime', partition_field_name: 'airline' },
{ function: 'min', field_name: 'responsetime', partition_field_name: 'airline' },
{ function: 'max', field_name: 'responsetime', partition_field_name: 'airline' },
],
},
data_description: { time_field: '@timestamp' },
analysis_limits: { model_memory_limit: '20mb' },
model_plot_config: { enabled: true },
};

// @ts-expect-error not full interface
const DATAFEED_CONFIG: Datafeed = {
datafeed_id: 'datafeed-fq_multi_1_ae',
indices: ['ft_farequote'],
job_id: 'fq_multi_1_ae',
query: { bool: { must: [{ match_all: {} }] } },
};

async function createMockJobs() {
await ml.api.createAnomalyDetectionJob(JOB_CONFIG);
await ml.api.createDatafeed(DATAFEED_CONFIG);
await ml.api.openAnomalyDetectionJob(JOB_CONFIG.job_id);
await ml.api.startDatafeed(DATAFEED_CONFIG.datafeed_id);
}

describe('GET model_management/nodes_overview', () => {
before(async () => {
await ml.testResources.setKibanaTimeZoneToUTC();
await ml.api.createTestTrainedModels('regression', 2);
await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/farequote');
await createMockJobs();
});

after(async () => {
await ml.api.closeAnomalyDetectionJob(JOB_CONFIG.job_id);
await ml.api.cleanMlIndices();
await ml.testResources.cleanMLSavedObjects();
});

it('returns nodes overview', async () => {
const { body, status } = await supertest
.get(`/internal/ml/model_management/nodes_overview`)
.auth(USER.ML_POWERUSER, ml.securityCommon.getPasswordForUser(USER.ML_POWERUSER))
.set(getCommonRequestHeader('1'));
ml.api.assertResponseStatusCode(200, status, body);

expect(body.nodes[0].roles).to.contain('ml');
expect(body.nodes[0].memory_overview.anomaly_detection.total).to.be.greaterThan(10000000);
expect(body.nodes[0].memory_overview.dfa_training.total).to.eql(0);
expect(body.nodes[0].memory_overview.trained_models.total).to.eql(0);
});

it('returns an error for the user with viewer permissions', async () => {
const { body, status } = await supertest
.get(`/internal/ml/model_management/nodes_overview`)
.auth(USER.ML_VIEWER, ml.securityCommon.getPasswordForUser(USER.ML_VIEWER))
.set(getCommonRequestHeader('1'));
ml.api.assertResponseStatusCode(403, status, body);
});

it('returns an error for unauthorized user', async () => {
const { body, status } = await supertest
.get(`/internal/ml/model_management/nodes_overview`)
.auth(USER.ML_UNAUTHORIZED, ml.securityCommon.getPasswordForUser(USER.ML_UNAUTHORIZED))
.set(getCommonRequestHeader('1'));
ml.api.assertResponseStatusCode(403, status, body);
});
});
};
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,16 @@ export default ({ getService }: FtrProviderContext) => {
await ml.api.cleanMlIndices();
});

it('returns trained model stats', async () => {
const { body, status } = await supertest
.get(`/internal/ml/trained_models/_stats`)
.auth(USER.ML_VIEWER, ml.securityCommon.getPasswordForUser(USER.ML_VIEWER))
.set(getCommonRequestHeader('1'));
ml.api.assertResponseStatusCode(200, status, body);

expect(body.count).to.eql(3);
});

it('returns trained model stats by id', async () => {
const { body, status } = await supertest
.get(`/internal/ml/trained_models/dfa_regression_model_n_0/_stats`)
Expand Down

0 comments on commit f7aeb6c

Please sign in to comment.