From 52b159618fb15fbedfa1e81ca8062b917313a083 Mon Sep 17 00:00:00 2001 From: Abhishek urs C J <43801187+Abhishk123@users.noreply.github.com> Date: Sat, 7 Sep 2024 01:08:23 +0530 Subject: [PATCH] feat: Sanity report logging (#213) * Added sanity report changes * Added testcontext changes * reverted the changes * Added testrunner changes * Added schema validation for all cases sanity * Added new schema format * Optimised getGenerateApiresult * Added minor changes * Addressed the review comments * Added UI changes * Added minor changes * Added minor changes: * Added unit tests for test runner * Added minor changes * Added additional unit test * Addressed the comments --- src/Test_Runner.js | 511 +++++------------ src/ValidationView.js | 30 +- src/constant.js | 4 +- src/pubsub/handlers/RunTestHandler.js | 2 +- src/utils/Utils.js | 1 - test/unit/test_runner.test.js | 796 +++++++++++++++++++------- 6 files changed, 753 insertions(+), 591 deletions(-) diff --git a/src/Test_Runner.js b/src/Test_Runner.js index e04ff204..cfd9c29c 100644 --- a/src/Test_Runner.js +++ b/src/Test_Runner.js @@ -42,9 +42,6 @@ const Validator = require('jsonschema').Validator; const validator = new Validator(); const logger = require('./utils/Logger')('Test_Runner.js'); const _ = require('lodash'); - -let validationResult; -let validationError = {}; const TAG = '[Test_Runner]: '; /** @@ -102,7 +99,6 @@ export class Test_Runner { // Start time of all API invocation const resultStartTime = new Date(); let suiteStartTime = new Date(); - let errorSchemaResult; errorSchemaValue = errorSchemaObject.errorSchema; // This is the list of validation Results for each api ,This is the list that will be used for creating the report @@ -160,11 +156,9 @@ export class Test_Runner { */ if (this.methodFilters.isMethodToBeExcluded(methodObj, communicationMode) || this.methodFilters.isRpcMethod(methodObj, invokedSdk, communicationMode)) { const obj = { - response: CONSTANTS.SKIPPED_MESSAGE, + error: CONSTANTS.SKIPPED_MESSAGE, param: undefined, - errorSchemaResult: undefined, methodWithExampleName: methodObj.name, - validationResult: {}, methodUuid: this.createUUID(), schemaData: schemaMap.schema, apiExecutionStartTime: apiExecutionStartTime, @@ -177,12 +171,10 @@ export class Test_Runner { overrideParamsFromTestData(method); for (let exampleIndex = 0; exampleIndex < method.examples.length; exampleIndex++) { let paramValues = []; - // The Subscribe methods are skipped for Transport, which is dynamically added from menubuilder - if (communicationMode == CONSTANTS.TRANSPORT) { - if (this.methodFilters.isSubscribeMethod(method.examples[exampleIndex]) || this.methodFilters.isSetMethod(method.examples[exampleIndex])) { - break; - } - } else if (this.methodFilters.isSetMethod(method.examples[exampleIndex])) { + if (this.methodFilters.isSubscribeMethod(method.examples[exampleIndex]) || this.methodFilters.isSetMethod(method.examples[exampleIndex])) { + break; + } + if (this.methodFilters.isSetMethod(method.examples[exampleIndex])) { continue; } @@ -197,18 +189,29 @@ export class Test_Runner { paramValues = example.params.map((p) => p.value); let result = null; + + // Overriding the schema with the below format + const schemaFormat = { + type: 'object', + properties: {}, + }; if (method.examples[exampleIndex].schema) { schemaMap = method.examples[exampleIndex]; } else { schemaMap = method.result; } - if (this.methodFilters.isExceptionMethod(methodObj.name, example.params)) { - if (method.examples[exampleIndex].schema) { - method.examples[exampleIndex].schema = errorSchemaValue; - } else { - method.result.schema = errorSchemaValue; - } + + // Check if the method is an exception method + const isExceptionMethod = this.methodFilters.isExceptionMethod(methodObj.name, example.params); + const propertyKey = isExceptionMethod ? 'error' : 'result'; + + // If the schema already has a "properties" field and does not have "error" or "result", override the schema + if ((schemaMap.schema.hasOwnProperty('properties') && !schemaMap.schema.properties.hasOwnProperty(propertyKey)) || !schemaMap.schema.hasOwnProperty('properties')) { + schemaFormat.properties[propertyKey] = isExceptionMethod ? errorSchemaValue : schemaMap.schema; + schemaFormat.required = [propertyKey]; + schemaMap.schema = schemaFormat; } + if (communicationMode == CONSTANTS.TRANSPORT) { const paramNames = method.params ? method.params.map((p) => p.name) : []; result = await this.apiInvoker(method.name, paramValues, executionMode, invokedSdk, paramNames); @@ -216,15 +219,17 @@ export class Test_Runner { result = await this.apiInvoker(method.name, paramValues, executionMode, invokedSdk); } - let schemaValidationResultForEachExample = method.examples[exampleIndex].schema ? validator.validate(result, method.examples[exampleIndex].schema) : validator.validate(result, method.result.schema); + const response = { result: result }; + let schemaValidationResultForEachExample = validator.validate(response, schemaMap.schema); + if (this.methodFilters.isEventMethod(methodObj)) { - logger.info(TAG + `${methodObj.name} Result => ${JSON.stringify(result)}`, 'northBoundSchemaValidationAndReportGeneration'); - if (result && typeof result.includes === 'function' && result.includes('Successful')) { - schemaValidationResultForEachExample = {}; + logger.info(TAG + `${methodObj.name} Result => ${JSON.stringify(response)}`, 'northBoundSchemaValidationAndReportGeneration'); + if (response && response.result && typeof response.result.includes === 'function' && response.result.includes('Successful')) { + schemaValidationResultForEachExample = { errors: [] }; } } const schemaValidationResultForEachExampleSet = { - response: result, + response: response, param: example.params, validationResult: schemaValidationResultForEachExample, methodWithExampleName: methodWithExampleName, @@ -235,56 +240,34 @@ export class Test_Runner { }; schemaValidationResultSet.push(schemaValidationResultForEachExampleSet); } catch (error) { + const errorResponse = { error: error }; let obj; - if (schemaMap == undefined && error == CONSTANTS.UNDEFINED_RESPONSE_MESSAGE) { - logger.debug('TestContext Debug: Error block on api execution - Acceptable No result: ' + error + ' for method: ' + methodWithExampleName, 'northBoundSchemaValidationAndReportGeneration'); - errorSchemaResult = false; + if (error instanceof Error) { + errorResponse.error = error.message; + } + logger.debug('TestContext Debug: Error block on api execution - has error message: ' + errorResponse.error + ' for method: ' + methodWithExampleName, 'northBoundSchemaValidationAndReportGeneration'); + // Doing schema validation for error response only if schema is present + if (schemaMap.schema) { + const schemaValidationResult = validator.validate(errorResponse, schemaMap.schema); obj = { - response: 'No result object - Acceptable', + error: errorResponse, param: example.params, - errorSchemaResult: errorSchemaResult, methodWithExampleName: methodWithExampleName, - validationResult: {}, + validationResult: schemaValidationResult, methodUuid: methodUuid, schemaData: schemaMap.schema, apiExecutionStartTime: apiExecutionStartTime, apiExecutionEndTime: apiExecutionEndTime, }; - } else if (error.responseError) { - logger.debug('TestContext Debug: Error block on api execution - has responseError: ' + error + ' for method: ' + methodWithExampleName, 'northBoundSchemaValidationAndReportGeneration'); - const err = error.responseError; - errorSchemaResult = true; + } else { obj = { - error: err, + error: errorResponse, param: example.params, - errorSchemaResult: errorSchemaResult, methodWithExampleName: methodWithExampleName, methodUuid: methodUuid, - schemaData: schemaMap.schema, apiExecutionStartTime: apiExecutionStartTime, apiExecutionEndTime: apiExecutionEndTime, }; - } else { - logger.debug('TestContext Debug: Error block on api execution - has error message: ' + error + ' for method: ' + methodWithExampleName, 'northBoundSchemaValidationAndReportGeneration'); - if (this.methodFilters.isExceptionMethod(methodObj.name, example.params)) { - obj = this.errorCheckForExemptedMethods(error, methodObj, methodWithExampleName, example, schemaMap, apiExecutionStartTime, apiExecutionEndTime); - } else { - let err = error; - if (typeof error == 'string') { - err = { code: 'CertError', message: error }; - } - errorSchemaResult = false; - obj = { - error: err, - param: example.params, - errorSchemaResult: errorSchemaResult, - methodWithExampleName: methodWithExampleName, - methodUuid: methodUuid, - schemaData: schemaMap.schema, - apiExecutionStartTime: apiExecutionStartTime, - apiExecutionEndTime: apiExecutionEndTime, - }; - } } schemaValidationResultSet.push(obj); } @@ -292,12 +275,8 @@ export class Test_Runner { } else { // Adding on more element to err Object to display method name on the screen for multiple testcases logger.debug('TestContext Debug: could not find example for method: ' + methodWithExampleName, 'northBoundSchemaValidationAndReportGeneration'); - const err = { - code: 'CertError', - message: 'Could not find an example for ' + method.name, - }; const obj = { - error: err, + error: 'Could not find an example for ' + method.name, param: null, methodWithExampleName: methodObj.name, methodUuid: methodUuid, @@ -530,11 +509,6 @@ export class Test_Runner { } else { if (response === undefined) { throw CONSTANTS.UNDEFINED_RESPONSE_MESSAGE; - } else if (response && response.error) { - const responseError = { - responseError: response, - }; - throw responseError; } } return response; @@ -836,301 +810,137 @@ export class Test_Runner { state: 'skipped', }; let convertedResponse = null; - let convertedValidationErr = null; - let methodName; - let errorSchemaResult; - let uuid; let testContext = null; - - const doesContainMethodNotFound = CONSTANTS.ERROR_LIST.find((i) => - JSON.stringify(result || '') - .toLowerCase() - .includes(i.toLowerCase()) - ); + let convertedError = null; + const methodWithExampleName = result.methodWithExampleName; + const uuid = result.methodUuid; + let parsedResponse = result.error ? result.error : result.response; + let doesErrorMessageContainMethodNotFound = false; const params = result.param; - if (result.error || doesContainMethodNotFound) { - let errorMessage, errorMessageLog; - if (result.error && result.error.message) { - errorMessage = result.error; - errorMessageLog = result.error.message; - } else { - const methodName = result.methodWithExampleName.split('.')[0] + '.' + result.methodWithExampleName.split('.')[1]; - if (this.methodFilters.isExceptionMethod(methodName, result.param)) { - errorMessage = result.error = `${CONSTANTS.WRONG_ERROR_MESSAGE_FORMAT}: ${JSON.stringify(result)}`; - errorMessageLog = `${CONSTANTS.WRONG_ERROR_MESSAGE_FORMAT}: ${JSON.stringify(result.error)}`; - } else { - errorMessage = result.error = `${CONSTANTS.WRONG_RESPONSE_MESSAGE_FORMAT}: ${JSON.stringify(result)}`; - errorMessageLog = `${CONSTANTS.WRONG_RESPONSE_MESSAGE_FORMAT}: ${JSON.stringify(result.error)}`; - } - } - const doesErrorMsgContainMethodNotFound = typeof errorMessageLog == 'string' && CONSTANTS.ERROR_LIST.find((i) => i.toLowerCase().includes(errorMessageLog.toLowerCase())); - - testContext = { - params: params, - result: null, - error: result.error, - }; - if (result.error.responseError) { - testContext.result = result.error.responseError; - testContext.error = null; - errorMessage = result.error.responseError.error; - } - - errorSchemaResult = result.errorSchemaResult; - if (errorMessage == undefined) { - errorMessage = 'undefined'; - } - - // for the below scenarios set the default status as failed + const methodName = result.methodWithExampleName.split('.')[0] + '.' + result.methodWithExampleName.split('.')[1]; + const isExceptionMethod = this.methodFilters.isExceptionMethod(methodName, params); + const schemaValidationResult = result.validationResult; + // Check if the error message contains "Method not found" + if (parsedResponse && parsedResponse.error && parsedResponse.error.message) { + doesErrorMessageContainMethodNotFound = CONSTANTS.ERROR_LIST.some((i) => + JSON.stringify(parsedResponse.error.message || '') + .toLowerCase() + .includes(i.toLowerCase()) + ); + } + testContext = { + params: params, + result: null, + error: null, + }; + if (!schemaValidationResult && result.error) { resultState = this.setResultState('failed'); - if (doesContainMethodNotFound && doesErrorMsgContainMethodNotFound) { - // When the underlying platform returns "Method not found" or "Not supported" in response.error.message. Certification suite will consider this as pending - errorMessage = JSON.stringify( - { - Schema: CONSTANTS.SCHEMA_CONTENT_SKIPPED, - Content: CONSTANTS.SCHEMA_CONTENT_SKIPPED, - Message: errorMessage, - params: params, - }, - null, - 1 - ); - // Disable SKIPPED and PENDING states in report based on flag - if (!process.env.CERTIFICATION) { - resultState = this.setResultState('pending'); - } - } else if (doesContainMethodNotFound && !doesErrorMsgContainMethodNotFound) { - // when the underlying platform returns "Method not found" or "Not supported" but in error. So not the correct error schema format. Certification will set the status as failed in this case - errorMessage = JSON.stringify({ Schema: CONSTANTS.FAILED, Content: CONSTANTS.FAILED, Actual: JSON.stringify(result) }, null, 1); - } else if ((errorSchemaResult && typeof errorMessage == 'string') || typeof errorMessage == 'object') { - errorMessage = JSON.stringify( - { - Schema: CONSTANTS.FAILED, - Content: CONSTANTS.SCHEMA_CONTENT_SKIPPED, - Actual: errorMessage, - Expected: schemaMap, - params: params, - }, - null, - 1 - ); - } else if (typeof errorMessage == 'string' || typeof errorMessage == 'object') { - errorMessage = JSON.stringify( - { - Schema: CONSTANTS.SCHEMA_CONTENT_SKIPPED, - Content: CONSTANTS.SCHEMA_CONTENT_SKIPPED, - Actual: errorMessage, - Expected: schemaMap, - params: params, - }, - null, - 1 - ); - } - - // isPass = false - - convertedResponse = errorMessage; - convertedValidationErr = result.error; - methodName = result.methodWithExampleName; - uuid = result.methodUuid; - if (typeof result.error.message == 'string' || Array.isArray(result.error.message) || typeof result.error.message == 'undefined') { - convertedValidationErr = { err: result.error }; - } - } else { - testContext = { - params: params, - result: result.response, - error: null, - }; - const schemaValidationResult = result.validationResult; - const contentPending = (schemaValidationResult && schemaValidationResult.contentPending) || false; - let response = result.response; - methodName = result.methodWithExampleName; - uuid = result.methodUuid; - - if (response === CONSTANTS.SKIPPED_MESSAGE) { + convertedError = { err: parsedResponse }; + // Skipping the test case if the response having skipped message + if (parsedResponse === CONSTANTS.SKIPPED_MESSAGE) { resultState = this.setResultState('skipped'); - convertedValidationErr = { err: CONSTANTS.NO_ERROR_FOUND }; - convertedResponse = JSON.stringify( - { - Schema: CONSTANTS.SCHEMA_CONTENT_SKIPPED, - Content: CONSTANTS.SCHEMA_CONTENT_SKIPPED, - Message: response, - }, - null, - 1 - ); - } else if (response === undefined || (schemaValidationResult.errors && schemaValidationResult.errors.length > 0)) { - resultState = this.setResultState('failed'); - validationError = schemaValidationResult.errors; - convertedValidationErr = validationError; - if (typeof validationError == 'string' || Array.isArray(validationError) || typeof result.response == 'undefined') { - convertedValidationErr = { err: validationError }; - } - if (response === undefined) { - if (hasContentValidationExecuted) { - convertedResponse = JSON.stringify( - { - Schema: CONSTANTS.FAILED, - Content: CONSTANTS.SCHEMA_CONTENT_SKIPPED, - Message: { - Actual: 'undefined', - Expected: schemaMap, - Message: CONSTANTS.UNDEFINED_RESPONSE_MESSAGE, - params: params, - }, - }, - null, - 1 - ); + convertedResponse = JSON.stringify({ [CONSTANTS.SCHEMA_VALIDATION]: CONSTANTS.SCHEMA_CONTENT_SKIPPED, Message: parsedResponse }, null, 1); + } else { + convertedResponse = JSON.stringify({ [CONSTANTS.SCHEMA_VALIDATION]: CONSTANTS.SCHEMA_CONTENT_SKIPPED, Message: parsedResponse, Response: null, Expected: schemaMap, params: params }, null, 1); + } + } else if (isExceptionMethod) { + resultState = this.setResultState('failed'); + // Check if parsed response contains an error + if (parsedResponse && parsedResponse.error) { + testContext.error = parsedResponse.error; + convertedError = { err: parsedResponse.error }; + // If it is an exception method, and not as per schema, fail the test case. + if (schemaValidationResult && schemaValidationResult.errors && schemaValidationResult.errors.length > 0) { + // Response did not have error or result + if (parsedResponse.error == CONSTANTS.UNDEFINED_RESPONSE_MESSAGE) { + testContext.error = null; + convertedResponse = JSON.stringify({ [CONSTANTS.SCHEMA_VALIDATION]: CONSTANTS.FAILED, Message: CONSTANTS.NO_RESULT_OR_ERROR_MESSAGE, Response: null, Expected: schemaMap, params: params }, null, 1); } else { - convertedResponse = JSON.stringify( - { - Schema: CONSTANTS.FAILED, - Content: CONSTANTS.PENDING, - Message: { - Actual: 'undefined', - Expected: schemaMap, - Message: CONSTANTS.UNDEFINED_RESPONSE_MESSAGE, - params: params, - }, - }, - null, - 1 - ); + convertedResponse = JSON.stringify({ [CONSTANTS.SCHEMA_VALIDATION]: CONSTANTS.FAILED, Message: 'Expected error, incorrect error format', Response: parsedResponse, Expected: schemaMap, params: params }, null, 1); } } else { - response = utils.censorData(methodObj.name, response); - testContext.result = response; - if (hasContentValidationExecuted) { - // Actual and Expected Schema/Content - if (schemaValidationResult.errors[0].message === 'Content is not valid') { - convertedResponse = JSON.stringify( - { - Schema: CONSTANTS.PASSED, - Content: CONSTANTS.FAILED, - Message: { - Actual: 'NA', - Expected: 'NA', - Error: schemaValidationResult.errors[0].message, - }, - params: params, - }, - null, - 1 - ); - } else { - convertedResponse = JSON.stringify( - { - Schema: CONSTANTS.FAILED, - Content: CONSTANTS.SCHEMA_CONTENT_SKIPPED, - Message: { Actual: response, Expected: schemaMap, Error: convertedValidationErr }, - params: params, - }, - null, - 1 - ); + // If error as per schema, error message contains method not found, marking the test case as pending or failed based on certification flag. + if (doesErrorMessageContainMethodNotFound) { + convertedResponse = JSON.stringify({ [CONSTANTS.SCHEMA_VALIDATION]: CONSTANTS.PASSED, Message: 'Method not implemented by platform', Response: parsedResponse, params: params }, null, 1); + // If the certification flag is enabled, fail the test case; otherwise, mark it as pending. + if (!process.env.CERTIFICATION) { + resultState = this.setResultState('pending'); } } else { - convertedResponse = JSON.stringify( - { - Schema: CONSTANTS.FAILED, - Content: CONSTANTS.PENDING, - Message: { - Actual: response, - Expected: schemaMap, - Error: schemaValidationResult.errors[0].message, - }, - params: params, - }, - null, - 1 - ); + // Exception method, and as per schema, marking the test case as passed. + resultState = this.setResultState('passed'); + convertedResponse = JSON.stringify({ [CONSTANTS.SCHEMA_VALIDATION]: CONSTANTS.PASSED, Message: 'Expected error, received error', Response: parsedResponse, params: params }, null, 1); } } } else { - // successfull validation - validationResult = CONSTANTS.PASSED; - // isPass = true - resultState = this.setResultState('passed'); - validationError = CONSTANTS.NO_ERROR_FOUND; - convertedValidationErr = validationError; - response = utils.censorData(methodObj.name, response); - testContext.result = response; - if (typeof validationError == 'string' || Array.isArray(validationError)) { - convertedValidationErr = { err: validationError }; - } - if (hasContentValidationExecuted && !contentPending) { - if (process.env.TESTCONTEXT) { - convertedResponse = JSON.stringify( - { - Schema: CONSTANTS.PASSED, - Content: CONSTANTS.PASSED, - Message: response, - params: params, - }, - null, - 1 - ); - } else { - convertedResponse = JSON.stringify({ Schema: CONSTANTS.PASSED, Content: CONSTANTS.PASSED, params: params }, null, 1); + // Censoring the response for the specific method + parsedResponse = utils.censorData(methodObj.name, parsedResponse.result); + testContext.result = parsedResponse; + convertedError = { err: CONSTANTS.NO_ERROR_FOUND }; + // Expecting an error, but received a result, marking the test case as failed. + convertedResponse = JSON.stringify({ [CONSTANTS.SCHEMA_VALIDATION]: CONSTANTS.FAILED, Message: 'Expected error, received result', Response: { result: parsedResponse }, Expected: schemaMap, params: params }, null, 1); + } + } else { + resultState = this.setResultState('passed'); + // Check if the response is an error + if (parsedResponse && parsedResponse.error) { + testContext.error = parsedResponse.result; + convertedError = { err: parsedResponse }; + resultState = this.setResultState('failed'); + // If error message contains method not found, marking the test case as pending or failed based on certification flag. + if (doesErrorMessageContainMethodNotFound) { + convertedResponse = JSON.stringify({ [CONSTANTS.SCHEMA_VALIDATION]: CONSTANTS.FAILED, Message: 'Method not implemented by platform', Response: parsedResponse, Expected: schemaMap, params: params }, null, 1); + // If the certification flag is enabled, fail the test case; otherwise, mark it as pending. + if (!process.env.CERTIFICATION) { + resultState = this.setResultState('pending'); } + } + // Response did not have error or result + else if (parsedResponse.error == CONSTANTS.UNDEFINED_RESPONSE_MESSAGE) { + testContext.error = null; + convertedResponse = JSON.stringify({ [CONSTANTS.SCHEMA_VALIDATION]: CONSTANTS.FAILED, Message: CONSTANTS.NO_RESULT_OR_ERROR_MESSAGE, Response: null, Expected: schemaMap, params: params }, null, 1); } else { - if (process.env.TESTCONTEXT) { - convertedResponse = JSON.stringify( - { - Schema: CONSTANTS.PASSED, - Content: CONSTANTS.PENDING, - Message: response, - params: params, - }, - null, - 1 - ); - } else { - convertedResponse = JSON.stringify({ Schema: CONSTANTS.PASSED, Content: CONSTANTS.PENDING, params: params }, null, 1); - } + // Expecting an result, but received an error, marking the test case as failed. + convertedResponse = JSON.stringify( + { [CONSTANTS.SCHEMA_VALIDATION]: CONSTANTS.FAILED, Message: 'Unexpected error encountered in the response', Response: parsedResponse, Expected: schemaMap, params: params }, + null, + 1 + ); } - if (response == 'No result object - Acceptable') { - if (process.env.TESTCONTEXT) { - convertedResponse = JSON.stringify( - { - Schema: CONSTANTS.SCHEMA_CONTENT_SKIPPED, - Content: CONSTANTS.SCHEMA_CONTENT_SKIPPED, - Message: response, - params: params, - }, - null, - 1 - ); - } else { - convertedResponse = JSON.stringify( - { - Schema: CONSTANTS.SCHEMA_CONTENT_SKIPPED, - Content: CONSTANTS.SCHEMA_CONTENT_SKIPPED, - params: params, - }, - null, - 1 - ); - } + } else { + // Censoring the response for the specific method + parsedResponse = utils.censorData(methodObj.name, parsedResponse.result); + testContext.result = parsedResponse; + convertedError = { err: CONSTANTS.NO_ERROR_FOUND }; + // If the response is not as per schema, marking the test case as failed else passed. + if (schemaValidationResult && schemaValidationResult.errors && schemaValidationResult.errors.length > 0) { + resultState = this.setResultState('failed'); + convertedResponse = JSON.stringify( + { [CONSTANTS.SCHEMA_VALIDATION]: CONSTANTS.FAILED, Message: schemaValidationResult.errors[0].stack, Response: { result: parsedResponse }, Expected: schemaMap, params: params }, + null, + 1 + ); + } else { + convertedResponse = JSON.stringify({ [CONSTANTS.SCHEMA_VALIDATION]: CONSTANTS.PASSED, Message: null, Response: { result: parsedResponse }, params: params }, null, 1); } } } + if (typeof convertedError == 'string' || Array.isArray(convertedError) || typeof convertedError == 'undefined') { + convertedError = { err: convertedError }; + } !process.env.TESTCONTEXT ? (testContext = null) : (testContext = JSON.stringify(testContext, null, 1)); const apiInvocationDuration = apiExecutionEndTime - apiExecutionStartTime; const apiValidationResult = { - title: methodName, + title: methodWithExampleName, fullTitle: methodObj.name, duration: apiInvocationDuration, state: resultState.state.toLowerCase(), pass: resultState.bool.passed, fail: resultState.bool.failed, code: convertedResponse, - err: convertedValidationErr, + err: convertedError, uuid: uuid, parentUUID: suitesUuid, timedOut: false, @@ -1217,37 +1027,4 @@ export class Test_Runner { logger.info('Error occured while generating sdk version', err, 'getFireboltVersionFromSDK'); } } - - errorCheckForExemptedMethods(error, methodObj, methodWithExampleName, example, schemaMap, apiExecutionStartTime, apiExecutionEndTime) { - let obj; - const NOT_SUPPORTED_ERROR_MESSAGES = ['Unsupported', 'Not supported', 'not supported']; - const errMessage = '{"code":' + error.code + ',"message":' + error.message + '}'; - const schemaValidationResult = errorSchemaCheck(error); - if (schemaValidationResult && schemaValidationResult.errors && schemaValidationResult.errors.length > 0) { - obj = { - error: error, - param: example.params, - errorSchemaResult: true, - methodWithExampleName: methodWithExampleName, - methodUuid: this.createUUID(), - schemaData: errorSchemaValue, - apiExecutionStartTime: apiExecutionStartTime, - apiExecutionEndTime: apiExecutionEndTime, - }; - } else { - NOT_SUPPORTED_ERROR_MESSAGES.some((errorMessage) => error.message.includes(errorMessage)); - obj = { - response: error, - param: example.params, - errorSchemaResult: undefined, - methodWithExampleName: methodWithExampleName, - validationResult: {}, - methodUuid: this.createUUID(), - schemaData: schemaMap.schema, - apiExecutionStartTime: apiExecutionStartTime, - apiExecutionEndTime: apiExecutionEndTime, - }; - } - return obj; - } } diff --git a/src/ValidationView.js b/src/ValidationView.js index d41d6c3e..a998aca0 100644 --- a/src/ValidationView.js +++ b/src/ValidationView.js @@ -122,7 +122,7 @@ export default class ValidationView extends lng.Component { }, color: 0xff123456, }, - ContentValidationStateText: { + Message: { x: 530, y: 200, w: 1920 - 700, @@ -198,7 +198,7 @@ export default class ValidationView extends lng.Component { this.tag('UpdateText').text = CONSTANTS.VALIDATION_SCROLLMESSAGE; this.tag('ApititleText').text = ''; this.tag('SchemaValidationStateText').text = ''; - this.tag('ContentValidationStateText').text = ''; + this.tag('Message').text = ''; this.tag('ValidationData').text = ''; } } else { @@ -230,7 +230,7 @@ export default class ValidationView extends lng.Component { const { err, fail, code } = _displayparms; this.tag('ValidationData').color = 0xff123456; let schemaValidationStateText = null, - contentValidationStateText = null, + message = null, validationData = null; if (code != undefined) { let codeObject = null, @@ -238,37 +238,31 @@ export default class ValidationView extends lng.Component { messageString = null; try { codeObject = JSON.parse(_displayparms.code); - messageString = codeObject.Message; - if (typeof codeObject.Message != 'string') { - messageString = JSON.stringify(codeObject.Message, null, 1); + messageString = codeObject.Response; + if (typeof codeObject.Response != 'string') { + messageString = JSON.stringify(codeObject.Response, null, 1); } isCodeTypeObject = true; } catch (err) { isCodeTypeObject = false; } if (isCodeTypeObject) { - schemaValidationStateText = CONSTANTS.SCHEMA_VALIDATION_STATUSMESSAGE + codeObject.Schema; - contentValidationStateText = CONSTANTS.CONTENT_VALIDATION_STATUSMESSAGE + codeObject.Content; - if (fail) { - validationData = CONSTANTS.ERROR_MESSAGE + messageString; - } else { - validationData = CONSTANTS.API_RESPONSE + messageString; - } + schemaValidationStateText = CONSTANTS.SCHEMA_VALIDATION_STATUSMESSAGE + codeObject['Schema Validation']; + message = 'Message: ' + codeObject.Message; + validationData = CONSTANTS.API_RESPONSE + messageString; } else { schemaValidationStateText = CONSTANTS.SCHEMA_VALIDATION_STATUSMESSAGE + CONSTANTS.SCHEMA_CONTENT_SKIPPED; - contentValidationStateText = CONSTANTS.CONTENT_VALIDATION_STATUSMESSAGE + CONSTANTS.SCHEMA_CONTENT_SKIPPED; - validationData = CONSTANTS.ERROR_MESSAGE + 'JSON parse failed (ValidationView)'; + validationData = CONSTANTS.API_RESPONSE + 'JSON parse failed (ValidationView)'; } } else { // Remove if not needed after testing schemaValidationStateText = CONSTANTS.SCHEMA_VALIDATION_STATUSMESSAGE + CONSTANTS.SCHEMA_CONTENT_SKIPPED; - contentValidationStateText = CONSTANTS.CONTENT_VALIDATION_STATUSMESSAGE + CONSTANTS.SCHEMA_CONTENT_SKIPPED; - validationData = CONSTANTS.ERROR_MESSAGE + 'Received response as undefined'; + validationData = CONSTANTS.API_RESPONSE + 'Received response as undefined'; } // Updating values in UI this.tag('ApititleText').text = CONSTANTS.API_TITLE + _displayparms.fullTitle; this.tag('SchemaValidationStateText').text = schemaValidationStateText; - this.tag('ContentValidationStateText').text = contentValidationStateText; + this.tag('Message').text = message; /* Schema data for some APIs are large enough to break the render engine. diff --git a/src/constant.js b/src/constant.js index 90f29a9e..dbc62c97 100644 --- a/src/constant.js +++ b/src/constant.js @@ -67,7 +67,7 @@ export const CONSTANTS = { CONTENT_ERROR: 'Content Error', RDK_SERVICES: 'org.rdk.', API_TITLE: 'API TITLE: ', - API_RESPONSE: 'API Response: ', + API_RESPONSE: 'Response: ', INVOKE_TEST_MESSAGE: '**** Click Invoke to run tests ****', VALIDATION_MESSAGE: '***** Validation Started ******', VALIDATION_SCROLLMESSAGE: "Scroll down through the menu's to view the result", @@ -172,4 +172,6 @@ export const CONSTANTS = { EXCLUDED_METHODS_FOR_MFOS: [], ...CONFIG_CONSTANTS, VERSIONS: 'Versions', + NO_RESULT_OR_ERROR_MESSAGE: 'No result or error in response. eg: {jsonrpc: "2.0", id: x }', + SCHEMA_VALIDATION: 'Schema Validation', }; diff --git a/src/pubsub/handlers/RunTestHandler.js b/src/pubsub/handlers/RunTestHandler.js index ad834fb5..19958478 100644 --- a/src/pubsub/handlers/RunTestHandler.js +++ b/src/pubsub/handlers/RunTestHandler.js @@ -71,7 +71,7 @@ export default class RunTestHandler extends BaseHandler { async getValidationReport(message) { const sdkMode = message.action; - process.env.COMMUNICATION_MODE = message.context.communicationMode; + process.env.COMMUNICATION_MODE = message.context.communicationMode ? message.context.communicationMode : CONSTANTS.TRANSPORT; const sdkInvokerInfo = new Test_Runner(); let validatedMenu; diff --git a/src/utils/Utils.js b/src/utils/Utils.js index 9df74b1c..3ba1d442 100644 --- a/src/utils/Utils.js +++ b/src/utils/Utils.js @@ -117,7 +117,6 @@ async function getschemaValidationDone(name, response, sdkType) { * @param response - response of the method */ function censorData(methodName, response) { - let json; try { const json = censorDataJson; if (methodName in json) { diff --git a/test/unit/test_runner.test.js b/test/unit/test_runner.test.js index 0bc92b14..9b531037 100644 --- a/test/unit/test_runner.test.js +++ b/test/unit/test_runner.test.js @@ -29,14 +29,14 @@ const Validator = require('jsonschema').Validator; let MOCK_OPEN_RPC_DOC = { methods: [ { - name: 'account.id', + name: 'Account.id', summary: 'Firebolt OpenRPC schema', params: [], result: { name: 'id', summary: 'the id', schema: { - type: 'object', + type: 'string', }, }, examples: [ @@ -51,7 +51,7 @@ let MOCK_OPEN_RPC_DOC = { ], }, { - name: 'account.uid', + name: 'Account.uid', summary: 'Gets a unique id for the current app & account', params: [], result: { @@ -72,8 +72,328 @@ let MOCK_OPEN_RPC_DOC = { }, ], }, + { + name: 'Device.id', + summary: 'Get the platform back-office device identifier', + params: [], + result: { + name: 'id', + summary: 'the id', + schema: { + type: 'string', + }, + }, + examples: [ + { + name: 'Default Example', + params: [], + result: { + name: 'Default Result', + value: '123', + }, + }, + ], + }, + { + name: 'Device.platform', + summary: 'Get the platform ID for this device', + params: [], + result: { + name: 'platformId', + summary: 'the platform ID', + schema: { + type: 'string', + }, + }, + examples: [ + { + name: 'Getting the platform ID', + params: [], + result: { + name: 'Default Result', + value: 'WPE', + }, + }, + ], + }, + { + name: 'Device.uid', + summary: 'Gets a unique id for the current app & device', + params: [], + result: { + name: 'uniqueId', + summary: 'a unique ID', + schema: { + type: 'string', + }, + }, + examples: [ + { + name: 'Getting the unique ID', + params: [], + result: { + name: 'Default Result', + value: 'ee6723b8-7ab3-462c-8d93-dbf61227998e', + }, + }, + ], + }, + { + name: 'Device.distributor', + summary: 'Get the distributor ID for this device', + params: [], + result: { + name: 'distributorId', + summary: 'the distributor ID', + schema: { + type: 'string', + }, + }, + examples: [ + { + name: 'Getting the distributor ID', + params: [], + result: { + name: 'Default Result', + value: 'Company', + }, + }, + ], + }, + { + name: 'Device.type', + summary: 'Get the device type', + params: [], + result: { + name: 'deviceType', + summary: 'the device type', + schema: { + type: 'string', + }, + }, + examples: [ + { + name: 'Getting the device type', + params: [], + result: { + name: 'Default Result', + value: 'STB', + }, + }, + ], + }, + { + name: 'Device.model', + summary: 'Get the device model', + params: [], + result: { + name: 'model', + summary: 'the device model', + schema: { + type: 'string', + }, + }, + examples: [ + { + name: 'Getting the device model', + params: [], + result: { + name: 'Default Result', + value: 'xi6', + }, + }, + ], + }, + { + name: 'Device.sku', + summary: 'Get the device sku', + params: [], + result: { + name: 'sku', + summary: 'the device sku', + schema: { + type: 'string', + }, + }, + examples: [ + { + name: 'Getting the device sku', + params: [], + result: { + name: 'Default Result', + value: 'AX061AEI', + }, + }, + ], + }, + { + name: 'Device.make', + summary: 'Get the device make', + params: [], + result: { + name: 'make', + summary: 'the device make', + schema: { + type: 'string', + }, + }, + }, + { + name: 'Device.hdcp', + summary: 'Get the supported HDCP profiles', + params: [], + tags: [ + { + name: 'property:readonly', + }, + { + name: 'capabilities', + 'x-uses': ['xrn:firebolt:capability:device:info'], + }, + ], + result: { + name: 'supportedHdcpProfiles', + summary: 'the supported HDCP profiles', + schema: { + type: 'object', + additionalProperties: { + type: 'boolean', + }, + }, + }, + examples: [ + { + name: 'Getting the supported HDCP profiles', + params: [], + result: { + name: 'Default Result', + value: { + 'hdcp1.4': true, + 'hdcp2.2': true, + }, + }, + }, + ], + }, + { + name: 'Accessibility.onClosedCaptionsSettingsChanged', + summary: "Get the user's preferred closed-captions settings", + params: [ + { + name: 'listen', + required: true, + schema: { + type: 'boolean', + }, + }, + ], + tags: [ + { + name: 'subscriber', + 'x-subscriber-for': 'Accessibility.closedCaptionsSettings', + }, + { + name: 'event', + 'x-alternative': 'closedCaptionsSettings', + }, + { + name: 'capabilities', + 'x-uses': ['xrn:firebolt:capability:accessibility:closedcaptions'], + }, + ], + result: { + name: 'closedCaptionsSettings', + summary: 'the closed captions settings', + schema: { + anyOf: [ + { + type: 'object', + required: ['event', 'listening'], + properties: { + event: { + type: 'string', + pattern: '[a-zA-Z]+\\.on[A-Z][a-zA-Z]+', + }, + listening: { + type: 'boolean', + }, + }, + additionalProperties: false, + }, + { + type: 'object', + required: ['enabled', 'styles'], + properties: { + enabled: { + type: 'boolean', + description: 'Whether or not closed-captions should be enabled by default', + }, + styles: { + type: 'object', + description: 'The default styles to use when displaying closed-captions', + }, + preferredLanguages: { + type: 'array', + items: { + type: 'string', + pattern: '^[a-z]{3}$', + }, + }, + }, + }, + ], + }, + }, + examples: [ + { + name: 'Getting the closed captions settings', + params: [ + { + name: 'listen', + value: true, + }, + ], + result: { + name: 'settings', + value: { + enabled: true, + styles: { + fontFamily: 'monospaced_sanserif', + fontSize: 1, + fontColor: '#ffffff', + fontEdge: 'none', + fontEdgeColor: '#7F7F7F', + fontOpacity: 100, + backgroundColor: '#000000', + backgroundOpacity: 100, + textAlign: 'center', + textAlignVertical: 'middle', + windowColor: 'white', + windowOpacity: 50, + }, + preferredLanguages: ['eng', 'spa'], + }, + }, + }, + ], + }, ], }; + +const mockResponses = { + 'Account.id': '123', + 'Account.uid': undefined, + 'Device.platform': { error: 'capability xrn:firebolt:capability:platformn is not supported' }, + 'Device.uid': { error: { code: -32601, message: 'Method not found' } }, + 'Device.distributor': { code: -50100, message: 'capability xrn:firebolt:capability:token:session is not supported' }, + 'Device.type': { error: { message: 'capability xrn:firebolt:capability:Localization.locality is not supported' } }, + 'Device.model': { error: { code: -32601, message: 'capability xrn:firebolt:capability:platformn is not supported' } }, + 'Device.sku': { error: { code: -32601, message: 'Method not found' } }, + 'Device.make': 'Arris', + 'De vice.hdcp': { 'hdcp1.4': true, 'hdcp2.2': true }, + 'Accessibility.onClosedCaptionsSettingsChanged': 'Successful accessibility.listen(closedCaptionsSettingsChanged)', +}; + const EXTERNAL_SDK_MOCK_OPEN_RPC_DOC = { methods: [ { @@ -122,13 +442,7 @@ const EXTERNAL_SDK_MOCK_OPEN_RPC_DOC = { }, ], }; -const MOCK_OPEN_RPC_RESPONSE = { id: 18, result: {}, jsonrpc: '2.0' }; -/** - * This is the definition of the structure used by Validation view - * to create the menu and also to show the result. - * This will be wrapped/transfomed to mocha awesome structure - * if the call is being invoked from Messnger.( Will be changed in FIRECET-72) - */ + const CUSTOM_REPORT_STRUCTURE_SCHEMA = { type: 'object', properties: { @@ -152,83 +466,87 @@ const CUSTOM_REPORT_STRUCTURE_SCHEMA = { }, }; -jest.mock('@apidevtools/json-schema-ref-parser', () => { - return { - dereference: () => { - return new Promise((resolve, reject) => { - if (!mockShouldDereferencerFail) { - resolve(MOCK_OPEN_RPC_DOC); - } else { - reject(new Error('Dereferencer Failure')); - } - }); - }, - }; -}); -/** - * mock object used to emulate the response from - * FireBoltExampleInvoker - */ +jest.mock('@apidevtools/json-schema-ref-parser', () => ({ + dereference: () => + new Promise((resolve, reject) => { + if (!mockShouldDereferencerFail) { + resolve(MOCK_OPEN_RPC_DOC); + } else { + reject(new Error('Dereferencer Failure')); + } + }), +})); + const mockFireboltExampleInvoker = { - invoke: () => {}, + invoke: jest.fn((sdk, methodName, params) => { + return returnMockResponse(methodName); + }), }; -jest.mock('../../src/FireboltExampleInvoker', () => { - return { - get: () => { - return mockFireboltExampleInvoker; - }, - }; -}); -jest.mock('@firebolt-js/sdk/dist/lib/Transport/index.mjs', () => { - return { - send: () => { - return {}; - }, - }; -}); -jest.mock('../../src/FireboltTransportInvoker', () => { - return { - get: () => { - return mockFireboltTransportInvoker; - }, - }; -}); -jest.mock('@firebolt-js/sdk', () => { - return { - Accessibility: {}, - Account: {}, - Advertising: {}, - Authentication: {}, - Device: {}, - Discovery: {}, - Keyboard: {}, - Lifecycle: { - ready: () => {}, - state: () => { - return 'initializing'; // dummy state value. - }, // returning a Lifecycle.state object - close: () => {}, - finish: () => {}, - }, - Localization: {}, - Metrics: {}, - Profile: {}, - Parameters: {}, - SecondScreen: {}, - }; -}); -jest.mock('../../src/pubsub/handlers/RegisterProviderHandler', () => { - return jest.fn().mockImplementation(() => ({ + +const mockFireboltTransportInvoker = { + invoke: jest.fn((methodName, params) => { + return returnMockResponse(methodName); + }), +}; + +function returnMockResponse(methodName) { + return new Promise((resolve, reject) => { + if (mockResponses.hasOwnProperty(methodName)) { + const response = mockResponses[methodName]; + if (response && response.error) { + reject(response.error); + } else { + resolve(response); + } + } else { + resolve({}); + } + }); +} + +jest.mock('../../src/FireboltExampleInvoker', () => ({ + get: () => mockFireboltExampleInvoker, +})); + +jest.mock('@firebolt-js/sdk/dist/lib/Transport/index.mjs', () => ({ + send: jest.fn().mockReturnValue({}), +})); + +jest.mock('../../src/FireboltTransportInvoker', () => ({ + get: () => mockFireboltTransportInvoker, +})); + +jest.mock('@firebolt-js/sdk', () => ({ + Accessibility: {}, + Account: {}, + Advertising: {}, + Authentication: {}, + Device: {}, + Discovery: {}, + Keyboard: {}, + Lifecycle: { + ready: jest.fn(), + state: jest.fn().mockReturnValue('initializing'), + close: jest.fn(), + finish: jest.fn(), + }, + Localization: {}, + Metrics: {}, + Profile: {}, + Parameters: {}, + SecondScreen: {}, +})); + +jest.mock('../../src/pubsub/handlers/RegisterProviderHandler', () => + jest.fn().mockImplementation(() => ({ handle: jest.fn().mockResolvedValue(JSON.stringify({ report: 'registered' })), - })); -}); -jest.mock('../../src/Toast', () => { - const eventEmitter = { - emit: jest.fn(), - }; + })) +); +jest.mock('../../src/Toast', () => { + const eventEmitter = { emit: jest.fn() }; return { - eventEmitter: eventEmitter, + eventEmitter, showToast: (toastMessage, toastState, toastRef) => { eventEmitter.emit('showToast', toastMessage, toastState, toastRef); }, @@ -241,146 +559,216 @@ const mockvalidationViewObj = { jest.mock('../../src/utils/Utils', () => ({ ...jest.requireActual('../../src/utils/Utils'), - pushReportToS3: () => { - return 'restApiUrl'; - }, - censorData: () => { - return 'censoredResponse'; - }, - dereferenceOpenRPC: (mode) => { - if (mode == 'externalsdk') { + pushReportToS3: jest.fn().mockReturnValue('restApiUrl'), + censorData: jest.fn((method, response) => { + return response; + }), + dereferenceOpenRPC: jest.fn((mode) => { + if (mode === 'externalsdk') { return [EXTERNAL_SDK_MOCK_OPEN_RPC_DOC, mode.toLowerCase()]; - } else if (mode == 'core' || mode == 'manage') { + } else if (mode === 'core' || mode === 'manage') { return [MOCK_OPEN_RPC_DOC, mode.toLowerCase()]; } - }, + }), })); -jest.mock('lodash', () => { - return { - cloneDeep: (value) => { - return value; - }, - }; -}); +jest.mock('lodash', () => ({ + cloneDeep: jest.fn((value) => value), +})); let mockShouldDereferencerFail = false; let runner; let result; const navigation = ''; +const INCLUDE_EVENT_METHODS = []; + +jest.mock('../../src/MethodFilters', () => ({ + __esModule: true, + default: jest.fn().mockImplementation(() => ({ + isExceptionMethod: jest.fn((methodName) => { + const exceptionMethods = ['Device.distributor', 'Device.type', 'Device.model', 'Device.sku']; + return exceptionMethods.includes(methodName); + }), + isMethodToBeExcluded: jest.fn((methodObject) => { + const excludedMethodsList = ['Device.hdcp']; + return excludedMethodsList.includes(methodObject.name); + }), + isRpcMethod: jest.fn(), + isSubscribeMethod: jest.fn(), + isSetMethod: jest.fn(), + shouldExcludeExample: jest.fn(), + isEventMethod: jest.fn((method) => { + let isEvent = false; + if (method.tags && INCLUDE_EVENT_METHODS.indexOf(method.name) === -1) { + method.tags.forEach((tag) => { + if (tag.name && tag.name === 'event') { + isEvent = true; + } + }); + } + return isEvent; + }), + })), +})); describe('Test_Runner test cases', () => { beforeEach(() => { runner = new Test_Runner(); - (runner.reportGenenration = jest.fn().mockImplementationOnce(() => { - return new Promise((resolve) => { - resolve(''); - }); - })), - (runner.invokeLifecycleAPI = jest.fn().mockImplementationOnce((tempParams) => { - if (tempParams.methodName == CONSTANTS.LIFECYCLE_METHOD_LIST[1]) { - return 'initializing'; - } else { - const mockLifecycleHistoryget = { _history: [{ someKey: 'someValue' }] }; - return mockLifecycleHistoryget; - } - })); + runner.reportGenenration = jest.fn().mockResolvedValue(''); + runner.invokeLifecycleAPI = jest.fn().mockImplementation((tempParams) => { + if (tempParams.methodName === CONSTANTS.LIFECYCLE_METHOD_LIST[1]) { + return 'initializing'; + } else { + return { _history: [{ someKey: 'someValue' }] }; + } + }); }); + describe('northBoundSchemaValidationAndReportGeneration Scenarios', () => { - test('Validate northBoundSchemaValidationAndReportGeneration(SDK) when OPEN RPC dereferece call fails', async () => { + test('should return empty result when dereference call fails for SDK', async () => { mockShouldDereferencerFail = true; - result = await runner.northBoundSchemaValidationAndReportGeneration('SDK', navigation, mockvalidationViewObj); + result = await runner.northBoundSchemaValidationAndReportGeneration('SDK'); /** when the dereference fails it should not execute any api and the result list will have 0 elements */ expect(result.length).toEqual(0); }); - test('Validate northBoundSchemaValidationAndReportGeneration(CORE) with example with valid response from FireboltExapmpleInvoker', async () => { - mockShouldDereferencerFail = false; - // Mock a valid response coming back from the Firebolt Invoker - mockFireboltExampleInvoker.invoke = () => Promise.resolve(MOCK_OPEN_RPC_RESPONSE); - result = await runner.northBoundSchemaValidationAndReportGeneration(CONSTANTS.CORE); - /** - * Since the mocked OPEN_RPC has 2 documents we will have 2 results - */ - expect(result.length).toEqual(2); - const v = new Validator(); - const schemaMapResult = v.validate(result[0], CUSTOM_REPORT_STRUCTURE_SCHEMA); - // This would make sure that the result json that is created is in valid strcuture - // we are not bothered about the value in the json. - expect(schemaMapResult.errors.length).toEqual(0); - /** - * Validate if the response title is populated correctly - */ - expect(result[1].fullTitle).toEqual('account.uid.Getting the unique ID'); - expect(result[0].state).toEqual('passed'); - }); - test('Validate northBoundSchemaValidationAndReportGeneration(CORE) with a invalid schema response coming back from FireboltExampleInvoker', async () => { - mockShouldDereferencerFail = false; - // Mock an invalid schema response coming back from the Firebolt Invoker - mockFireboltExampleInvoker.invoke = () => Promise.resolve(null); + describe('northBoundSchemaValidationAndReportGeneration Scenarios for CORE', () => { + beforeAll(async () => { + result = await runner.northBoundSchemaValidationAndReportGeneration(CONSTANTS.CORE); + }); + test('should return valid response for Account.id API', async () => { + const extractedResult = result.find((obj) => obj.title === 'Account.id'); + extractedResult.code = JSON.parse(extractedResult.code); + expect(extractedResult.code['Schema Validation']).toEqual('Passed'); + expect(extractedResult.code.Response.result).toBeDefined(); + }); - result = await runner.northBoundSchemaValidationAndReportGeneration(CONSTANTS.CORE); - /** - * Since the mocked OPEN_RPC has 2 documents we will have 2 results - */ - expect(result.length).toEqual(2); - // Both results should have resulted in schema validation failure - expect(result[0].state).toEqual('failed'); + test('should fail for Account.uid API due to undefined response', async () => { + const extractedResult = result.find((obj) => obj.title === 'Account.uid'); + extractedResult.code = JSON.parse(extractedResult.code); + expect(extractedResult.code['Schema Validation']).toEqual('Failed'); + expect(extractedResult.code.Response).toBeNull(); + expect(extractedResult.code.Message).toContain('No result or error in response.'); + }); + + test('should handle different type of response for Device.id API', async () => { + const extractedResult = result.find((obj) => obj.title === 'Device.id'); + extractedResult.code = JSON.parse(extractedResult.code); + expect(extractedResult.code['Schema Validation']).toEqual('Failed'); + expect(extractedResult.code.Response.result).toBeDefined(); + expect(extractedResult.code.Message).toContain('instance.result is not of a type(s) string'); + }); + + test('should handle unexpected error for Device.platform API', async () => { + const extractedResult = result.find((obj) => obj.title === 'Device.platform'); + extractedResult.code = JSON.parse(extractedResult.code); + expect(extractedResult.code['Schema Validation']).toEqual('Failed'); + expect(extractedResult.code.Response.error).toBeDefined(); + expect(extractedResult.code.Message).toContain('Unexpected error encountered in the response'); + }); + + test('should handle unexpected error: "method not implemented" for Device.uid API', async () => { + const extractedResult = result.find((obj) => obj.title === 'Device.uid'); + extractedResult.code = JSON.parse(extractedResult.code); + expect(extractedResult.code['Schema Validation']).toEqual('Failed'); + expect(extractedResult.code.Response.error).toBeDefined(); + expect(extractedResult.code.Message).toContain('Method not implemented by platform'); + }); + + test('should handle expecting error but received result for Device.distributor API', async () => { + const extractedResult = result.find((obj) => obj.title === 'Device.distributor'); + extractedResult.code = JSON.parse(extractedResult.code); + expect(extractedResult.code['Schema Validation']).toEqual('Failed'); + expect(extractedResult.code.Response.result).toBeDefined(); + expect(extractedResult.code.Message).toContain('Expected error, received result'); + }); + + test('should handle expecting error but incorrect error format for Device.type API', async () => { + const extractedResult = result.find((obj) => obj.title === 'Device.type'); + extractedResult.code = JSON.parse(extractedResult.code); + expect(extractedResult.code['Schema Validation']).toEqual('Failed'); + expect(extractedResult.code.Response.error).toBeDefined(); + expect(extractedResult.code.Message).toContain('Expected error, incorrect error format'); + }); + + test('should handle expected error but received error for Device.model API', async () => { + const extractedResult = result.find((obj) => obj.title === 'Device.model'); + extractedResult.code = JSON.parse(extractedResult.code); + expect(extractedResult.code['Schema Validation']).toEqual('Passed'); + expect(extractedResult.code.Response.error).toBeDefined(); + expect(extractedResult.code.Message).toContain('Expected error, received error'); + }); + + test('should handle method not implemented error for Device.sku API', async () => { + const extractedResult = result.find((obj) => obj.title === 'Device.sku'); + extractedResult.code = JSON.parse(extractedResult.code); + expect(extractedResult.code['Schema Validation']).toEqual('Passed'); + expect(extractedResult.code.Response.error).toBeDefined(); + expect(extractedResult.code.Message).toContain('Method not implemented by platform'); + }); + + test('should skip validation for Device.make API due to missing example', async () => { + const extractedResult = result.find((obj) => obj.title === 'Device.make'); + extractedResult.code = JSON.parse(extractedResult.code); + expect(extractedResult.code['Schema Validation']).toEqual('Skipped'); + expect(extractedResult.code.Response).toBeNull(); + expect(extractedResult.code.Message).toContain('Could not find an example for Device.make'); + }); + + test('should return valid response for Accessibility.onClosedCaptionsSettingsChanged Event', async () => { + const extractedResult = result.find((obj) => obj.title === 'Accessibility.onClosedCaptionsSettingsChanged'); + extractedResult.code = JSON.parse(extractedResult.code); + expect(extractedResult.code['Schema Validation']).toEqual('Passed'); + expect(extractedResult.code.Response.result).toBeDefined(); + }); }); - test('Validate northBoundSchemaValidationAndReportGeneration when invalid mode is passed', async () => { - /** when the invalid mode is passed dereference fails it should not execute any api */ + + test('should return error when invalid mode is passed', async () => { mockShouldDereferencerFail = false; result = await runner.northBoundSchemaValidationAndReportGeneration('undefined', navigation, mockvalidationViewObj); expect(result.error).toEqual(CONSTANTS.NOTPERFORMED); }); - test('Validate northBoundSchemaValidationAndReportGeneration(CORE) with no example with valid response from FireboltExapmpleInvoker', async () => { - mockShouldDereferencerFail = false; - // Mock a valid response coming back from the Firebolt Invoker - // Overriding MOCK_OPEN_RPC_DOC value for device module test - + test('should handle when mode as passes as an arry', async () => { MOCK_OPEN_RPC_DOC = { methods: [ { - name: 'device.uid', + name: 'Account.id', summary: 'Firebolt OpenRPC schema', params: [], result: { - name: 'OpenRPC Schema', + name: 'id', + summary: 'the id', schema: { - type: 'object', + type: 'string', }, }, + examples: [ + { + name: 'Default Example', + params: [], + result: { + name: 'Default Result', + value: '123', + }, + }, + ], }, ], }; - mockFireboltExampleInvoker.invoke = () => Promise.resolve(MOCK_OPEN_RPC_RESPONSE); - - result = await runner.northBoundSchemaValidationAndReportGeneration(CONSTANTS.CORE); - /** - * Since the mocked OPEN_RPC has 2 documents we will have 2 results - */ - expect(result.length).toEqual(1); - const v = new Validator(); - const schemaMapResult = v.validate(result[0], CUSTOM_REPORT_STRUCTURE_SCHEMA); - // This would make sure that the result json that is created is in valid strcuture - // we are not bothered about the value in the json. - expect(schemaMapResult.errors.length).toEqual(0); - /** - * Validate if the response title is populated correctly - */ - expect(result[0].fullTitle).toEqual('device.uid'); - expect(result[0].code).toContain('Could not find an example for device.uid'); + result = await runner.northBoundSchemaValidationAndReportGeneration([CONSTANTS.CORE]); + const extractedResult = result.find((obj) => obj.title === 'Account.id'); + extractedResult.code = JSON.parse(extractedResult.code); + expect(extractedResult.code['Schema Validation']).toEqual('Passed'); + expect(extractedResult.code.Response.result).toBeDefined(); }); - test('Validate northBoundSchemaValidationAndReportGeneration when module is device (schema and content validation from library)', async () => { - /** when the module is device, content and schema validation would be done externally */ - mockShouldDereferencerFail = false; - // Overriding MOCK_OPEN_RPC_DOC value for device module test + + test('should handle when communication mode is Transport', async () => { MOCK_OPEN_RPC_DOC = { methods: [ { - name: 'device.id', - summary: 'Get the platform back-office device identifier', + name: 'Account.id', + summary: 'Firebolt OpenRPC schema', params: [], result: { name: 'id', @@ -402,43 +790,45 @@ describe('Test_Runner test cases', () => { }, ], }; - mockFireboltExampleInvoker.invoke = () => Promise.resolve(MOCK_OPEN_RPC_RESPONSE); - result = await runner.northBoundSchemaValidationAndReportGeneration(CONSTANTS.CORE, navigation, mockvalidationViewObj); - const parsedCode = JSON.parse(result[0].code); - // Schema validation is expected to fail as MOCK_OPEN_RPC_RESPONSE is not in the expected schema for device module - expect(parsedCode.Schema).toEqual('Failed'); - // Content validation will be skipped when schema validation fails - expect(parsedCode.Content).toEqual('Pending'); - - // Reverting MOCK_OPEN_RPC_DOC to previous value + process.env.COMMUNICATION_MODE = 'Transport'; + result = await runner.northBoundSchemaValidationAndReportGeneration([CONSTANTS.CORE]); + const extractedResult = result.find((obj) => obj.title === 'Account.id'); + extractedResult.code = JSON.parse(extractedResult.code); + expect(extractedResult.code['Schema Validation']).toEqual('Passed'); + expect(extractedResult.code.Response.result).toBeDefined(); + }); + + test('should handle when schema is missing from openRpc', async () => { MOCK_OPEN_RPC_DOC = { methods: [ { - name: 'rpc.discover', + name: 'Account.id', summary: 'Firebolt OpenRPC schema', params: [], result: { - name: 'OpenRPC Schema', - schema: { - type: 'object', - }, + name: 'id', + summary: 'the id', }, - }, - { - name: 'account.id', - summary: 'Firebolt OpenRPC schema', - params: [], - result: { - name: 'OpenRPC Schema', - schema: { - type: 'object', + examples: [ + { + name: 'Default Example', + params: [], + result: { + name: 'Default Result', + value: '123', + }, }, - }, + ], }, ], }; + result = await runner.northBoundSchemaValidationAndReportGeneration([CONSTANTS.CORE]); + const extractedResult = result.find((obj) => obj.title === 'Account.id'); + extractedResult.code = JSON.parse(extractedResult.code); + expect(extractedResult.code['Schema Validation']).toEqual('Skipped'); }); }); + describe('UUID Generation Validation', () => { let firstuuid, seconduuid; test('validate uuid when generated two uuid are different', () => { @@ -520,7 +910,7 @@ describe('Test_Runner test cases', () => { test('Validate northBoundSchemaValidationAndReportGeneration(externalSDK) with example with valid response from FireboltExapmpleInvoker', async () => { mockShouldDereferencerFail = false; // Mock a valid response coming back from the Firebolt Invoker - mockFireboltExampleInvoker.invoke = () => Promise.resolve(MOCK_OPEN_RPC_RESPONSE); + mockFireboltExampleInvoker.invoke = () => Promise.resolve({ id: 18, result: {}, jsonrpc: '2.0' }); result = await runner.northBoundSchemaValidationAndReportGeneration('externalSDK'); /**