diff --git a/README.md b/README.md index c1f44aca..43747269 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ It has the following features - - [FCA URL deployed and available in the S3](#fca-url-deployed-and-available-in-the-s3) - [Below are the steps to run FCA in local system](#below-are-the-steps-to-run-fca-in-local-system) - [Supported ways of Execution](#supported-ways-of-execution) +- [Sanity Suite Flow](./docs/index.md) - [Supported targets](#supported-targets) - [Supported Modes of execution](#supported-modes-of-execution) - [Supported validations](#supported-validations) diff --git a/docs/images/generateAPIValidaionResult.png b/docs/images/generateAPIValidaionResult.png new file mode 100644 index 00000000..e90874fc Binary files /dev/null and b/docs/images/generateAPIValidaionResult.png differ diff --git a/docs/images/testRunnerFlow.png b/docs/images/testRunnerFlow.png new file mode 100644 index 00000000..fb7fa77d Binary files /dev/null and b/docs/images/testRunnerFlow.png differ diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 00000000..29084e0b --- /dev/null +++ b/docs/index.md @@ -0,0 +1,7 @@ +# Sanity Suite Flow chart + +## Implementation +![Implementation Diagram](./images/testRunnerFlow.png) + +## Generating Report Logs +![Manual Trigger Diagram](./images/generateAPIValidaionResult.png) diff --git a/package.json b/package.json index 37b69544..fa05ad9b 100644 --- a/package.json +++ b/package.json @@ -1,12 +1,12 @@ { - "version": "1.2.0", + "version": "1.3.0", "name": "firebolt-certification", "description": "Reference App to demonstrate Firebolt APIs and Lifecycle", "dependencies": { "@apidevtools/json-schema-ref-parser": "^9.0.9", - "@firebolt-js/discovery-sdk": "1.2.0", - "@firebolt-js/manage-sdk": "1.2.0", - "@firebolt-js/sdk": "1.2.0", + "@firebolt-js/discovery-sdk": "1.3.0", + "@firebolt-js/manage-sdk": "1.3.0", + "@firebolt-js/sdk": "1.3.0", "@lightningjs/core": "2.11.0", "@lightningjs/sdk": "^5.0.1", "@lightningjs/ui-components": "^2.2.2", diff --git a/src/EventInvocation.js b/src/EventInvocation.js index 63297d1a..40c29421 100644 --- a/src/EventInvocation.js +++ b/src/EventInvocation.js @@ -138,7 +138,7 @@ export class EventInvocation { let schemaValidationResult = {}; let schemaValidationStatus = CONSTANTS.PASS; if (message.params.isNotSupportedApi == true) { - schemaValidationResult = errorSchemaCheck(listenerResponse, process.env.COMMUNICATION_MODE); + schemaValidationResult = errorSchemaCheck(listenerResponse); schemaValidationStatus = CONSTANTS.FAIL; } registrationResponse['eventListenerSchemaResult'] = { @@ -153,7 +153,7 @@ export class EventInvocation { } registrationResponse['eventListenerResponse'] = { result: null, error: listenerResponse }; // In case of error, validate error against errorschema - const schemaValidationResult = errorSchemaCheck(listenerResponse, process.env.COMMUNICATION_MODE); + const schemaValidationResult = errorSchemaCheck(listenerResponse); if (schemaValidationResult && schemaValidationResult.errors && schemaValidationResult.errors.length > 0) { registrationResponse['eventListenerSchemaResult'] = { status: CONSTANTS.FAIL, diff --git a/src/MethodInvoker.js b/src/MethodInvoker.js index d5d2305b..476a9579 100644 --- a/src/MethodInvoker.js +++ b/src/MethodInvoker.js @@ -103,7 +103,7 @@ export class MethodInvoker { if (process.env.STANDALONE == true) { // if the method is not supported and it gives a valid response, validate against errorschema instead of api schema if (message.params.isNotSupportedApi == true && response != undefined) { - schemaValidationResult = errorSchemaCheck(response, process.env.COMMUNICATION_MODE); + schemaValidationResult = errorSchemaCheck(response); } } } catch (error) { @@ -135,7 +135,7 @@ export class MethodInvoker { let apiResponse, responseCode, schemaValidationStatus; if (err) { apiResponse = { result: null, error: err }; - schemaValidationResult = errorSchemaCheck(err, process.env.COMMUNICATION_MODE); + schemaValidationResult = errorSchemaCheck(err); if (schemaValidationResult && schemaValidationResult.errors && schemaValidationResult.errors.length > 0) { if (err.message != undefined && CONSTANTS.ERROR_LIST.includes(err.message)) { responseCode = CONSTANTS.STATUS_CODE[3]; diff --git a/src/Test_Runner.js b/src/Test_Runner.js index 812ff2d9..cfd9c29c 100644 --- a/src/Test_Runner.js +++ b/src/Test_Runner.js @@ -36,15 +36,12 @@ const utils = require('./utils/Utils'); import LifecycleHistory from './LifeCycleHistory'; import { Device } from '@firebolt-js/sdk'; import { MODULE_MAP } from './FireboltExampleInvoker'; -import errorSchema from './source/errorSchema.json'; +import errorSchemaObject from './source/errorSchema.json'; const $RefParser = require('@apidevtools/json-schema-ref-parser'); const Validator = require('jsonschema').Validator; const validator = new Validator(); const logger = require('./utils/Logger')('Test_Runner.js'); const _ = require('lodash'); - -let validationResult; -let validationError = {}; const TAG = '[Test_Runner]: '; /** @@ -52,13 +49,13 @@ const TAG = '[Test_Runner]: '; */ let execMode; let invokedSdk; -let errorSchemaBasedOnMode; +let errorSchemaValue; /* Start and End time of API invocation */ let apiExecutionEndTime; -let apiExecutionStartTime = new Date(); +let apiExecutionStartTime; export class Test_Runner { /** @@ -102,8 +99,7 @@ export class Test_Runner { // Start time of all API invocation const resultStartTime = new Date(); let suiteStartTime = new Date(); - let errorSchemaResult; - errorSchemaBasedOnMode = process.env.COMMUNICATION_MODE == CONSTANTS.TRANSPORT ? errorSchema[CONSTANTS.ERROR_SCHEMA_TRANSPORT] : errorSchema[CONSTANTS.ERROR_SCHEMA_SDK]; + errorSchemaValue = errorSchemaObject.errorSchema; // This is the list of validation Results for each api ,This is the list that will be used for creating the report for (const executionMode of execModes) { @@ -133,6 +129,8 @@ export class Test_Runner { // traverse the json data inside loop to get methodname & properties for (let methodIndex = 0; this.dereferenceSchemaList != undefined && methodIndex < this.dereferenceSchemaList.methods.length; methodIndex++) { const module = this.dereferenceSchemaList.methods[methodIndex].name.split('.')[0]; + apiExecutionEndTime = 0; + apiExecutionStartTime = 0; let methodUuid = this.createUUID(); // uuid of this method const method = this.dereferenceSchemaList.methods[methodIndex]; const methodObj = this.dereferenceSchemaList.methods[methodIndex]; @@ -158,13 +156,13 @@ export class Test_Runner { */ if (this.methodFilters.isMethodToBeExcluded(methodObj, communicationMode) || this.methodFilters.isRpcMethod(methodObj, invokedSdk, communicationMode)) { const obj = { - response: CONSTANTS.SKIPPED_MESSAGE, + error: CONSTANTS.SKIPPED_MESSAGE, param: undefined, - errorSchemaResult: undefined, methodWithExampleName: methodObj.name, - validationResult: {}, methodUuid: this.createUUID(), schemaData: schemaMap.schema, + apiExecutionStartTime: apiExecutionStartTime, + apiExecutionEndTime: apiExecutionEndTime, }; schemaValidationResultSet.push(obj); } else if (!this.methodFilters.isRpcMethod(methodObj, invokedSdk, communicationMode)) { @@ -173,12 +171,10 @@ export class Test_Runner { overrideParamsFromTestData(method); for (let exampleIndex = 0; exampleIndex < method.examples.length; exampleIndex++) { let paramValues = []; - // The Subscribe methods are skipped for Transport, which is dynamically added from menubuilder - if (communicationMode == CONSTANTS.TRANSPORT) { - if (this.methodFilters.isSubscribeMethod(method.examples[exampleIndex]) || this.methodFilters.isSetMethod(method.examples[exampleIndex])) { - break; - } - } else if (this.methodFilters.isSetMethod(method.examples[exampleIndex])) { + if (this.methodFilters.isSubscribeMethod(method.examples[exampleIndex]) || this.methodFilters.isSetMethod(method.examples[exampleIndex])) { + break; + } + if (this.methodFilters.isSetMethod(method.examples[exampleIndex])) { continue; } @@ -193,18 +189,29 @@ export class Test_Runner { paramValues = example.params.map((p) => p.value); let result = null; + + // Overriding the schema with the below format + const schemaFormat = { + type: 'object', + properties: {}, + }; if (method.examples[exampleIndex].schema) { schemaMap = method.examples[exampleIndex]; } else { schemaMap = method.result; } - if (this.methodFilters.isExceptionMethod(methodObj.name, example.params)) { - if (method.examples[exampleIndex].schema) { - method.examples[exampleIndex].schema = errorSchemaBasedOnMode; - } else { - method.result.schema = errorSchemaBasedOnMode; - } + + // Check if the method is an exception method + const isExceptionMethod = this.methodFilters.isExceptionMethod(methodObj.name, example.params); + const propertyKey = isExceptionMethod ? 'error' : 'result'; + + // If the schema already has a "properties" field and does not have "error" or "result", override the schema + if ((schemaMap.schema.hasOwnProperty('properties') && !schemaMap.schema.properties.hasOwnProperty(propertyKey)) || !schemaMap.schema.hasOwnProperty('properties')) { + schemaFormat.properties[propertyKey] = isExceptionMethod ? errorSchemaValue : schemaMap.schema; + schemaFormat.required = [propertyKey]; + schemaMap.schema = schemaFormat; } + if (communicationMode == CONSTANTS.TRANSPORT) { const paramNames = method.params ? method.params.map((p) => p.name) : []; result = await this.apiInvoker(method.name, paramValues, executionMode, invokedSdk, paramNames); @@ -212,67 +219,55 @@ export class Test_Runner { result = await this.apiInvoker(method.name, paramValues, executionMode, invokedSdk); } - let schemaValidationResultForEachExample = method.examples[exampleIndex].schema ? validator.validate(result, method.examples[exampleIndex].schema) : validator.validate(result, method.result.schema); + const response = { result: result }; + let schemaValidationResultForEachExample = validator.validate(response, schemaMap.schema); + if (this.methodFilters.isEventMethod(methodObj)) { - logger.info(TAG + `${methodObj.name} Result => ${JSON.stringify(result)}`, 'northBoundSchemaValidationAndReportGeneration'); - if (result && typeof result.includes === 'function' && result.includes('Successful')) { - schemaValidationResultForEachExample = {}; + logger.info(TAG + `${methodObj.name} Result => ${JSON.stringify(response)}`, 'northBoundSchemaValidationAndReportGeneration'); + if (response && response.result && typeof response.result.includes === 'function' && response.result.includes('Successful')) { + schemaValidationResultForEachExample = { errors: [] }; } } const schemaValidationResultForEachExampleSet = { - response: result, + response: response, param: example.params, validationResult: schemaValidationResultForEachExample, methodWithExampleName: methodWithExampleName, methodUuid: methodUuid, schemaData: schemaMap.schema, + apiExecutionStartTime: apiExecutionStartTime, + apiExecutionEndTime: apiExecutionEndTime, }; schemaValidationResultSet.push(schemaValidationResultForEachExampleSet); } catch (error) { + const errorResponse = { error: error }; let obj; - if (schemaMap == undefined && error == CONSTANTS.UNDEFINED_RESPONSE_MESSAGE) { - logger.debug('TestContext Debug: Error block on api execution - Acceptable No result: ' + error + ' for method: ' + methodWithExampleName, 'northBoundSchemaValidationAndReportGeneration'); - errorSchemaResult = false; + if (error instanceof Error) { + errorResponse.error = error.message; + } + logger.debug('TestContext Debug: Error block on api execution - has error message: ' + errorResponse.error + ' for method: ' + methodWithExampleName, 'northBoundSchemaValidationAndReportGeneration'); + // Doing schema validation for error response only if schema is present + if (schemaMap.schema) { + const schemaValidationResult = validator.validate(errorResponse, schemaMap.schema); obj = { - response: 'No result object - Acceptable', + error: errorResponse, param: example.params, - errorSchemaResult: errorSchemaResult, methodWithExampleName: methodWithExampleName, - validationResult: {}, + validationResult: schemaValidationResult, methodUuid: methodUuid, schemaData: schemaMap.schema, + apiExecutionStartTime: apiExecutionStartTime, + apiExecutionEndTime: apiExecutionEndTime, }; - } else if (error.responseError) { - logger.debug('TestContext Debug: Error block on api execution - has responseError: ' + error + ' for method: ' + methodWithExampleName, 'northBoundSchemaValidationAndReportGeneration'); - const err = error.responseError; - errorSchemaResult = true; + } else { obj = { - error: err, + error: errorResponse, param: example.params, - errorSchemaResult: errorSchemaResult, methodWithExampleName: methodWithExampleName, methodUuid: methodUuid, - schemaData: schemaMap.schema, + apiExecutionStartTime: apiExecutionStartTime, + apiExecutionEndTime: apiExecutionEndTime, }; - } else { - logger.debug('TestContext Debug: Error block on api execution - has error message: ' + error + ' for method: ' + methodWithExampleName, 'northBoundSchemaValidationAndReportGeneration'); - if (this.methodFilters.isExceptionMethod(methodObj.name, example.params)) { - obj = this.errorCheckForExemptedMethods(error, methodObj, methodWithExampleName, example, schemaMap); - } else { - let err = error; - if (typeof error == 'string') { - err = { code: 'CertError', message: error }; - } - errorSchemaResult = false; - obj = { - error: err, - param: example.params, - errorSchemaResult: errorSchemaResult, - methodWithExampleName: methodWithExampleName, - methodUuid: methodUuid, - schemaData: schemaMap.schema, - }; - } } schemaValidationResultSet.push(obj); } @@ -280,16 +275,14 @@ export class Test_Runner { } else { // Adding on more element to err Object to display method name on the screen for multiple testcases logger.debug('TestContext Debug: could not find example for method: ' + methodWithExampleName, 'northBoundSchemaValidationAndReportGeneration'); - const err = { - code: 'CertError', - message: 'Could not find an example for ' + method.name, - }; const obj = { - error: err, + error: 'Could not find an example for ' + method.name, param: null, methodWithExampleName: methodObj.name, methodUuid: methodUuid, schemaData: schemaMap.schema, + apiExecutionStartTime: apiExecutionStartTime, + apiExecutionEndTime: apiExecutionEndTime, }; schemaValidationResultSet.push(obj); } @@ -308,7 +301,9 @@ export class Test_Runner { schema = schemaValidationRes.schemaData; } delete schemaValidationRes.schemaData; - const apiValidationResult = this.generateAPIValidaionResult(schemaValidationRes, methodObj, apiExecutionStartTime, apiExecutionEndTime, suitesUuid, hasContentValidationExecuted, schema); + const executionStartTime = schemaValidationRes.apiExecutionStartTime; + const executionEndTime = schemaValidationRes.apiExecutionEndTime; + const apiValidationResult = this.generateAPIValidaionResult(schemaValidationRes, methodObj, executionStartTime, executionEndTime, suitesUuid, hasContentValidationExecuted, schema); if (apiValidationResult.pass) { successList.push(apiValidationResult.uuid); } else if (apiValidationResult.skipped) { @@ -488,12 +483,13 @@ export class Test_Runner { sdk = invokedSdk; } executionMode = executionMode.toUpperCase(); - apiExecutionStartTime = new Date(); // api execution start time if (executionMode.includes(CONSTANTS.MANAGE) || executionMode.includes(CONSTANTS.CORE) || executionMode.includes(CONSTANTS.DISCOVERY)) { + apiExecutionStartTime = new Date(); // api execution start time [response, err] = paramsArray ? await handleAsyncFunction(FireboltTransportInvoker.get().invoke(method, params, paramsArray)) : await handleAsyncFunction(FireboltExampleInvoker.get().invoke(sdk, method, params, null, paramsArray)); + apiExecutionEndTime = new Date(); // api execution end time // To handle event response trimming observed when events invoked via transport mode if (response) { if (response.hasOwnProperty('event') == true) { @@ -507,18 +503,12 @@ export class Test_Runner { } else { response = CONSTANTS.NOTPERFORMED; } - apiExecutionEndTime = new Date(); // api execution end time // If an error happens while invoking the function throw error if (err) { throw err; } else { if (response === undefined) { throw CONSTANTS.UNDEFINED_RESPONSE_MESSAGE; - } else if (response && response.error) { - const responseError = { - responseError: response, - }; - throw responseError; } } return response; @@ -820,301 +810,137 @@ export class Test_Runner { state: 'skipped', }; let convertedResponse = null; - let convertedValidationErr = null; - let methodName; - let errorSchemaResult; - let uuid; let testContext = null; - - const doesContainMethodNotFound = CONSTANTS.ERROR_LIST.find((i) => - JSON.stringify(result || '') - .toLowerCase() - .includes(i.toLowerCase()) - ); + let convertedError = null; + const methodWithExampleName = result.methodWithExampleName; + const uuid = result.methodUuid; + let parsedResponse = result.error ? result.error : result.response; + let doesErrorMessageContainMethodNotFound = false; const params = result.param; - if (result.error || doesContainMethodNotFound) { - let errorMessage, errorMessageLog; - if (result.error && result.error.message) { - errorMessage = result.error; - errorMessageLog = result.error.message; - } else { - const methodName = result.methodWithExampleName.split('.')[0] + '.' + result.methodWithExampleName.split('.')[1]; - if (this.methodFilters.isExceptionMethod(methodName, result.param)) { - errorMessage = result.error = `${CONSTANTS.WRONG_ERROR_MESSAGE_FORMAT}: ${JSON.stringify(result)}`; - errorMessageLog = `${CONSTANTS.WRONG_ERROR_MESSAGE_FORMAT}: ${JSON.stringify(result.error)}`; - } else { - errorMessage = result.error = `${CONSTANTS.WRONG_RESPONSE_MESSAGE_FORMAT}: ${JSON.stringify(result)}`; - errorMessageLog = `${CONSTANTS.WRONG_RESPONSE_MESSAGE_FORMAT}: ${JSON.stringify(result.error)}`; - } - } - const doesErrorMsgContainMethodNotFound = typeof errorMessageLog == 'string' && CONSTANTS.ERROR_LIST.find((i) => i.toLowerCase().includes(errorMessageLog.toLowerCase())); - - testContext = { - params: params, - result: null, - error: result.error, - }; - if (result.error.responseError) { - testContext.result = result.error.responseError; - testContext.error = null; - errorMessage = result.error.responseError.error; - } - - errorSchemaResult = result.errorSchemaResult; - if (errorMessage == undefined) { - errorMessage = 'undefined'; - } - - // for the below scenarios set the default status as failed + const methodName = result.methodWithExampleName.split('.')[0] + '.' + result.methodWithExampleName.split('.')[1]; + const isExceptionMethod = this.methodFilters.isExceptionMethod(methodName, params); + const schemaValidationResult = result.validationResult; + // Check if the error message contains "Method not found" + if (parsedResponse && parsedResponse.error && parsedResponse.error.message) { + doesErrorMessageContainMethodNotFound = CONSTANTS.ERROR_LIST.some((i) => + JSON.stringify(parsedResponse.error.message || '') + .toLowerCase() + .includes(i.toLowerCase()) + ); + } + testContext = { + params: params, + result: null, + error: null, + }; + if (!schemaValidationResult && result.error) { resultState = this.setResultState('failed'); - if (doesContainMethodNotFound && doesErrorMsgContainMethodNotFound) { - // When the underlying platform returns "Method not found" or "Not supported" in response.error.message. Certification suite will consider this as pending - errorMessage = JSON.stringify( - { - Schema: CONSTANTS.SCHEMA_CONTENT_SKIPPED, - Content: CONSTANTS.SCHEMA_CONTENT_SKIPPED, - Message: errorMessage, - params: params, - }, - null, - 1 - ); - // Disable SKIPPED and PENDING states in report based on flag - if (!process.env.CERTIFICATION) { - resultState = this.setResultState('pending'); - } - } else if (doesContainMethodNotFound && !doesErrorMsgContainMethodNotFound) { - // when the underlying platform returns "Method not found" or "Not supported" but in error. So not the correct error schema format. Certification will set the status as failed in this case - errorMessage = JSON.stringify({ Schema: CONSTANTS.FAILED, Content: CONSTANTS.FAILED, Actual: JSON.stringify(result) }, null, 1); - } else if ((errorSchemaResult && typeof errorMessage == 'string') || typeof errorMessage == 'object') { - errorMessage = JSON.stringify( - { - Schema: CONSTANTS.FAILED, - Content: CONSTANTS.SCHEMA_CONTENT_SKIPPED, - Actual: errorMessage, - Expected: schemaMap, - params: params, - }, - null, - 1 - ); - } else if (typeof errorMessage == 'string' || typeof errorMessage == 'object') { - errorMessage = JSON.stringify( - { - Schema: CONSTANTS.SCHEMA_CONTENT_SKIPPED, - Content: CONSTANTS.SCHEMA_CONTENT_SKIPPED, - Actual: errorMessage, - Expected: schemaMap, - params: params, - }, - null, - 1 - ); - } - - // isPass = false - - convertedResponse = errorMessage; - convertedValidationErr = result.error; - methodName = result.methodWithExampleName; - uuid = result.methodUuid; - if (typeof result.error.message == 'string' || Array.isArray(result.error.message) || typeof result.error.message == 'undefined') { - convertedValidationErr = { err: result.error }; - } - } else { - testContext = { - params: params, - result: result.response, - error: null, - }; - const schemaValidationResult = result.validationResult; - const contentPending = (schemaValidationResult && schemaValidationResult.contentPending) || false; - let response = result.response; - methodName = result.methodWithExampleName; - uuid = result.methodUuid; - - if (response === CONSTANTS.SKIPPED_MESSAGE) { + convertedError = { err: parsedResponse }; + // Skipping the test case if the response having skipped message + if (parsedResponse === CONSTANTS.SKIPPED_MESSAGE) { resultState = this.setResultState('skipped'); - convertedValidationErr = { err: CONSTANTS.NO_ERROR_FOUND }; - convertedResponse = JSON.stringify( - { - Schema: CONSTANTS.SCHEMA_CONTENT_SKIPPED, - Content: CONSTANTS.SCHEMA_CONTENT_SKIPPED, - Message: response, - }, - null, - 1 - ); - } else if (response === undefined || (schemaValidationResult.errors && schemaValidationResult.errors.length > 0)) { - resultState = this.setResultState('failed'); - validationError = schemaValidationResult.errors; - convertedValidationErr = validationError; - if (typeof validationError == 'string' || Array.isArray(validationError) || typeof result.response == 'undefined') { - convertedValidationErr = { err: validationError }; - } - if (response === undefined) { - if (hasContentValidationExecuted) { - convertedResponse = JSON.stringify( - { - Schema: CONSTANTS.FAILED, - Content: CONSTANTS.SCHEMA_CONTENT_SKIPPED, - Message: { - Actual: 'undefined', - Expected: schemaMap, - Message: CONSTANTS.UNDEFINED_RESPONSE_MESSAGE, - params: params, - }, - }, - null, - 1 - ); + convertedResponse = JSON.stringify({ [CONSTANTS.SCHEMA_VALIDATION]: CONSTANTS.SCHEMA_CONTENT_SKIPPED, Message: parsedResponse }, null, 1); + } else { + convertedResponse = JSON.stringify({ [CONSTANTS.SCHEMA_VALIDATION]: CONSTANTS.SCHEMA_CONTENT_SKIPPED, Message: parsedResponse, Response: null, Expected: schemaMap, params: params }, null, 1); + } + } else if (isExceptionMethod) { + resultState = this.setResultState('failed'); + // Check if parsed response contains an error + if (parsedResponse && parsedResponse.error) { + testContext.error = parsedResponse.error; + convertedError = { err: parsedResponse.error }; + // If it is an exception method, and not as per schema, fail the test case. + if (schemaValidationResult && schemaValidationResult.errors && schemaValidationResult.errors.length > 0) { + // Response did not have error or result + if (parsedResponse.error == CONSTANTS.UNDEFINED_RESPONSE_MESSAGE) { + testContext.error = null; + convertedResponse = JSON.stringify({ [CONSTANTS.SCHEMA_VALIDATION]: CONSTANTS.FAILED, Message: CONSTANTS.NO_RESULT_OR_ERROR_MESSAGE, Response: null, Expected: schemaMap, params: params }, null, 1); } else { - convertedResponse = JSON.stringify( - { - Schema: CONSTANTS.FAILED, - Content: CONSTANTS.PENDING, - Message: { - Actual: 'undefined', - Expected: schemaMap, - Message: CONSTANTS.UNDEFINED_RESPONSE_MESSAGE, - params: params, - }, - }, - null, - 1 - ); + convertedResponse = JSON.stringify({ [CONSTANTS.SCHEMA_VALIDATION]: CONSTANTS.FAILED, Message: 'Expected error, incorrect error format', Response: parsedResponse, Expected: schemaMap, params: params }, null, 1); } } else { - response = utils.censorData(methodObj.name, response); - testContext.result = response; - if (hasContentValidationExecuted) { - // Actual and Expected Schema/Content - if (schemaValidationResult.errors[0].message === 'Content is not valid') { - convertedResponse = JSON.stringify( - { - Schema: CONSTANTS.PASSED, - Content: CONSTANTS.FAILED, - Message: { - Actual: 'NA', - Expected: 'NA', - Error: schemaValidationResult.errors[0].message, - }, - params: params, - }, - null, - 1 - ); - } else { - convertedResponse = JSON.stringify( - { - Schema: CONSTANTS.FAILED, - Content: CONSTANTS.SCHEMA_CONTENT_SKIPPED, - Message: { Actual: response, Expected: schemaMap, Error: convertedValidationErr }, - params: params, - }, - null, - 1 - ); + // If error as per schema, error message contains method not found, marking the test case as pending or failed based on certification flag. + if (doesErrorMessageContainMethodNotFound) { + convertedResponse = JSON.stringify({ [CONSTANTS.SCHEMA_VALIDATION]: CONSTANTS.PASSED, Message: 'Method not implemented by platform', Response: parsedResponse, params: params }, null, 1); + // If the certification flag is enabled, fail the test case; otherwise, mark it as pending. + if (!process.env.CERTIFICATION) { + resultState = this.setResultState('pending'); } } else { - convertedResponse = JSON.stringify( - { - Schema: CONSTANTS.FAILED, - Content: CONSTANTS.PENDING, - Message: { - Actual: response, - Expected: schemaMap, - Error: schemaValidationResult.errors[0].message, - }, - params: params, - }, - null, - 1 - ); + // Exception method, and as per schema, marking the test case as passed. + resultState = this.setResultState('passed'); + convertedResponse = JSON.stringify({ [CONSTANTS.SCHEMA_VALIDATION]: CONSTANTS.PASSED, Message: 'Expected error, received error', Response: parsedResponse, params: params }, null, 1); } } } else { - // successfull validation - validationResult = CONSTANTS.PASSED; - // isPass = true - resultState = this.setResultState('passed'); - validationError = CONSTANTS.NO_ERROR_FOUND; - convertedValidationErr = validationError; - response = utils.censorData(methodObj.name, response); - testContext.result = response; - if (typeof validationError == 'string' || Array.isArray(validationError)) { - convertedValidationErr = { err: validationError }; - } - if (hasContentValidationExecuted && !contentPending) { - if (process.env.TESTCONTEXT) { - convertedResponse = JSON.stringify( - { - Schema: CONSTANTS.PASSED, - Content: CONSTANTS.PASSED, - Message: response, - params: params, - }, - null, - 1 - ); - } else { - convertedResponse = JSON.stringify({ Schema: CONSTANTS.PASSED, Content: CONSTANTS.PASSED, params: params }, null, 1); + // Censoring the response for the specific method + parsedResponse = utils.censorData(methodObj.name, parsedResponse.result); + testContext.result = parsedResponse; + convertedError = { err: CONSTANTS.NO_ERROR_FOUND }; + // Expecting an error, but received a result, marking the test case as failed. + convertedResponse = JSON.stringify({ [CONSTANTS.SCHEMA_VALIDATION]: CONSTANTS.FAILED, Message: 'Expected error, received result', Response: { result: parsedResponse }, Expected: schemaMap, params: params }, null, 1); + } + } else { + resultState = this.setResultState('passed'); + // Check if the response is an error + if (parsedResponse && parsedResponse.error) { + testContext.error = parsedResponse.result; + convertedError = { err: parsedResponse }; + resultState = this.setResultState('failed'); + // If error message contains method not found, marking the test case as pending or failed based on certification flag. + if (doesErrorMessageContainMethodNotFound) { + convertedResponse = JSON.stringify({ [CONSTANTS.SCHEMA_VALIDATION]: CONSTANTS.FAILED, Message: 'Method not implemented by platform', Response: parsedResponse, Expected: schemaMap, params: params }, null, 1); + // If the certification flag is enabled, fail the test case; otherwise, mark it as pending. + if (!process.env.CERTIFICATION) { + resultState = this.setResultState('pending'); } + } + // Response did not have error or result + else if (parsedResponse.error == CONSTANTS.UNDEFINED_RESPONSE_MESSAGE) { + testContext.error = null; + convertedResponse = JSON.stringify({ [CONSTANTS.SCHEMA_VALIDATION]: CONSTANTS.FAILED, Message: CONSTANTS.NO_RESULT_OR_ERROR_MESSAGE, Response: null, Expected: schemaMap, params: params }, null, 1); } else { - if (process.env.TESTCONTEXT) { - convertedResponse = JSON.stringify( - { - Schema: CONSTANTS.PASSED, - Content: CONSTANTS.PENDING, - Message: response, - params: params, - }, - null, - 1 - ); - } else { - convertedResponse = JSON.stringify({ Schema: CONSTANTS.PASSED, Content: CONSTANTS.PENDING, params: params }, null, 1); - } + // Expecting an result, but received an error, marking the test case as failed. + convertedResponse = JSON.stringify( + { [CONSTANTS.SCHEMA_VALIDATION]: CONSTANTS.FAILED, Message: 'Unexpected error encountered in the response', Response: parsedResponse, Expected: schemaMap, params: params }, + null, + 1 + ); } - if (response == 'No result object - Acceptable') { - if (process.env.TESTCONTEXT) { - convertedResponse = JSON.stringify( - { - Schema: CONSTANTS.SCHEMA_CONTENT_SKIPPED, - Content: CONSTANTS.SCHEMA_CONTENT_SKIPPED, - Message: response, - params: params, - }, - null, - 1 - ); - } else { - convertedResponse = JSON.stringify( - { - Schema: CONSTANTS.SCHEMA_CONTENT_SKIPPED, - Content: CONSTANTS.SCHEMA_CONTENT_SKIPPED, - params: params, - }, - null, - 1 - ); - } + } else { + // Censoring the response for the specific method + parsedResponse = utils.censorData(methodObj.name, parsedResponse.result); + testContext.result = parsedResponse; + convertedError = { err: CONSTANTS.NO_ERROR_FOUND }; + // If the response is not as per schema, marking the test case as failed else passed. + if (schemaValidationResult && schemaValidationResult.errors && schemaValidationResult.errors.length > 0) { + resultState = this.setResultState('failed'); + convertedResponse = JSON.stringify( + { [CONSTANTS.SCHEMA_VALIDATION]: CONSTANTS.FAILED, Message: schemaValidationResult.errors[0].stack, Response: { result: parsedResponse }, Expected: schemaMap, params: params }, + null, + 1 + ); + } else { + convertedResponse = JSON.stringify({ [CONSTANTS.SCHEMA_VALIDATION]: CONSTANTS.PASSED, Message: null, Response: { result: parsedResponse }, params: params }, null, 1); } } } + if (typeof convertedError == 'string' || Array.isArray(convertedError) || typeof convertedError == 'undefined') { + convertedError = { err: convertedError }; + } !process.env.TESTCONTEXT ? (testContext = null) : (testContext = JSON.stringify(testContext, null, 1)); const apiInvocationDuration = apiExecutionEndTime - apiExecutionStartTime; const apiValidationResult = { - title: methodName, + title: methodWithExampleName, fullTitle: methodObj.name, duration: apiInvocationDuration, state: resultState.state.toLowerCase(), pass: resultState.bool.passed, fail: resultState.bool.failed, code: convertedResponse, - err: convertedValidationErr, + err: convertedError, uuid: uuid, parentUUID: suitesUuid, timedOut: false, @@ -1201,33 +1027,4 @@ export class Test_Runner { logger.info('Error occured while generating sdk version', err, 'getFireboltVersionFromSDK'); } } - - errorCheckForExemptedMethods(error, methodObj, methodWithExampleName, example, schemaMap) { - let obj; - const NOT_SUPPORTED_ERROR_MESSAGES = ['Unsupported', 'Not supported', 'not supported']; - const errMessage = '{"code":' + error.code + ',"message":' + error.message + '}'; - const schemaValidationResult = errorSchemaCheck(error, process.env.COMMUNICATION_MODE); - if (schemaValidationResult && schemaValidationResult.errors && schemaValidationResult.errors.length > 0) { - obj = { - error: error, - param: example.params, - errorSchemaResult: true, - methodWithExampleName: methodWithExampleName, - methodUuid: this.createUUID(), - schemaData: errorSchemaBasedOnMode, - }; - } else { - NOT_SUPPORTED_ERROR_MESSAGES.some((errorMessage) => error.message.includes(errorMessage)); - obj = { - response: error, - param: example.params, - errorSchemaResult: undefined, - methodWithExampleName: methodWithExampleName, - validationResult: {}, - methodUuid: this.createUUID(), - schemaData: schemaMap.schema, - }; - } - return obj; - } } diff --git a/src/ValidationView.js b/src/ValidationView.js index d41d6c3e..a998aca0 100644 --- a/src/ValidationView.js +++ b/src/ValidationView.js @@ -122,7 +122,7 @@ export default class ValidationView extends lng.Component { }, color: 0xff123456, }, - ContentValidationStateText: { + Message: { x: 530, y: 200, w: 1920 - 700, @@ -198,7 +198,7 @@ export default class ValidationView extends lng.Component { this.tag('UpdateText').text = CONSTANTS.VALIDATION_SCROLLMESSAGE; this.tag('ApititleText').text = ''; this.tag('SchemaValidationStateText').text = ''; - this.tag('ContentValidationStateText').text = ''; + this.tag('Message').text = ''; this.tag('ValidationData').text = ''; } } else { @@ -230,7 +230,7 @@ export default class ValidationView extends lng.Component { const { err, fail, code } = _displayparms; this.tag('ValidationData').color = 0xff123456; let schemaValidationStateText = null, - contentValidationStateText = null, + message = null, validationData = null; if (code != undefined) { let codeObject = null, @@ -238,37 +238,31 @@ export default class ValidationView extends lng.Component { messageString = null; try { codeObject = JSON.parse(_displayparms.code); - messageString = codeObject.Message; - if (typeof codeObject.Message != 'string') { - messageString = JSON.stringify(codeObject.Message, null, 1); + messageString = codeObject.Response; + if (typeof codeObject.Response != 'string') { + messageString = JSON.stringify(codeObject.Response, null, 1); } isCodeTypeObject = true; } catch (err) { isCodeTypeObject = false; } if (isCodeTypeObject) { - schemaValidationStateText = CONSTANTS.SCHEMA_VALIDATION_STATUSMESSAGE + codeObject.Schema; - contentValidationStateText = CONSTANTS.CONTENT_VALIDATION_STATUSMESSAGE + codeObject.Content; - if (fail) { - validationData = CONSTANTS.ERROR_MESSAGE + messageString; - } else { - validationData = CONSTANTS.API_RESPONSE + messageString; - } + schemaValidationStateText = CONSTANTS.SCHEMA_VALIDATION_STATUSMESSAGE + codeObject['Schema Validation']; + message = 'Message: ' + codeObject.Message; + validationData = CONSTANTS.API_RESPONSE + messageString; } else { schemaValidationStateText = CONSTANTS.SCHEMA_VALIDATION_STATUSMESSAGE + CONSTANTS.SCHEMA_CONTENT_SKIPPED; - contentValidationStateText = CONSTANTS.CONTENT_VALIDATION_STATUSMESSAGE + CONSTANTS.SCHEMA_CONTENT_SKIPPED; - validationData = CONSTANTS.ERROR_MESSAGE + 'JSON parse failed (ValidationView)'; + validationData = CONSTANTS.API_RESPONSE + 'JSON parse failed (ValidationView)'; } } else { // Remove if not needed after testing schemaValidationStateText = CONSTANTS.SCHEMA_VALIDATION_STATUSMESSAGE + CONSTANTS.SCHEMA_CONTENT_SKIPPED; - contentValidationStateText = CONSTANTS.CONTENT_VALIDATION_STATUSMESSAGE + CONSTANTS.SCHEMA_CONTENT_SKIPPED; - validationData = CONSTANTS.ERROR_MESSAGE + 'Received response as undefined'; + validationData = CONSTANTS.API_RESPONSE + 'Received response as undefined'; } // Updating values in UI this.tag('ApititleText').text = CONSTANTS.API_TITLE + _displayparms.fullTitle; this.tag('SchemaValidationStateText').text = schemaValidationStateText; - this.tag('ContentValidationStateText').text = contentValidationStateText; + this.tag('Message').text = message; /* Schema data for some APIs are large enough to break the render engine. diff --git a/src/constant.js b/src/constant.js index 0d98cab4..dbc62c97 100644 --- a/src/constant.js +++ b/src/constant.js @@ -67,7 +67,7 @@ export const CONSTANTS = { CONTENT_ERROR: 'Content Error', RDK_SERVICES: 'org.rdk.', API_TITLE: 'API TITLE: ', - API_RESPONSE: 'API Response: ', + API_RESPONSE: 'Response: ', INVOKE_TEST_MESSAGE: '**** Click Invoke to run tests ****', VALIDATION_MESSAGE: '***** Validation Started ******', VALIDATION_SCROLLMESSAGE: "Scroll down through the menu's to view the result", @@ -172,6 +172,6 @@ export const CONSTANTS = { EXCLUDED_METHODS_FOR_MFOS: [], ...CONFIG_CONSTANTS, VERSIONS: 'Versions', - ERROR_SCHEMA_TRANSPORT: 'errorSchemaTransport', - ERROR_SCHEMA_SDK: 'errorSchemaSDK', + NO_RESULT_OR_ERROR_MESSAGE: 'No result or error in response. eg: {jsonrpc: "2.0", id: x }', + SCHEMA_VALIDATION: 'Schema Validation', }; diff --git a/src/pubsub/handlers/RunTestHandler.js b/src/pubsub/handlers/RunTestHandler.js index ad834fb5..19958478 100644 --- a/src/pubsub/handlers/RunTestHandler.js +++ b/src/pubsub/handlers/RunTestHandler.js @@ -71,7 +71,7 @@ export default class RunTestHandler extends BaseHandler { async getValidationReport(message) { const sdkMode = message.action; - process.env.COMMUNICATION_MODE = message.context.communicationMode; + process.env.COMMUNICATION_MODE = message.context.communicationMode ? message.context.communicationMode : CONSTANTS.TRANSPORT; const sdkInvokerInfo = new Test_Runner(); let validatedMenu; diff --git a/src/source/errorSchema.json b/src/source/errorSchema.json index 0f7e4011..7f115d7d 100644 --- a/src/source/errorSchema.json +++ b/src/source/errorSchema.json @@ -1,57 +1,23 @@ { - "errorSchemaSDK": { - "oneOf": [ - { - "type": "object", - "additionalProperties": false, - "required": [ - "code", - "message" - ], - "properties": { - "code": { - "title": "errorObjectCode", - "description": "A Number that indicates the error type that occurred. This MUST be an integer. The error codes from and including -32768 to -32000 are reserved for pre-defined errors. These pre-defined errors SHOULD be assumed to be returned from any JSON-RPC api.", - "type": "integer" - }, - "message": { - "title": "errorObjectMessage", - "description": "A String providing a short description of the error. The message SHOULD be limited to a concise single sentence.", - "type": "string" - }, - "data": { - "title": "errorObjectData", - "description": "A Primitive or Structured value that contains additional information about the error. This may be omitted. The value of this member is defined by the Server (e.g. detailed error information, nested errors etc.)." - } - } - }, - { - "type": "string" - } - ] - }, - "errorSchemaTransport": { - "type": "object", - "additionalProperties": false, - "required": [ - "code", - "message" - ], - "properties": { - "code": { - "title": "errorObjectCode", - "description": "A Number that indicates the error type that occurred. This MUST be an integer. The error codes from and including -32768 to -32000 are reserved for pre-defined errors. These pre-defined errors SHOULD be assumed to be returned from any JSON-RPC api.", - "type": "integer" - }, - "message": { - "title": "errorObjectMessage", - "description": "A String providing a short description of the error. The message SHOULD be limited to a concise single sentence.", - "type": "string" - }, - "data": { - "title": "errorObjectData", - "description": "A Primitive or Structured value that contains additional information about the error. This may be omitted. The value of this member is defined by the Server (e.g. detailed error information, nested errors etc.)." - } - } + "errorSchema": { + "type": "object", + "additionalProperties": false, + "required": ["code", "message"], + "properties": { + "code": { + "title": "errorObjectCode", + "description": "A Number that indicates the error type that occurred. This MUST be an integer. The error codes from and including -32768 to -32000 are reserved for pre-defined errors. These pre-defined errors SHOULD be assumed to be returned from any JSON-RPC api.", + "type": "integer" + }, + "message": { + "title": "errorObjectMessage", + "description": "A String providing a short description of the error. The message SHOULD be limited to a concise single sentence.", + "type": "string" + }, + "data": { + "title": "errorObjectData", + "description": "A Primitive or Structured value that contains additional information about the error. This may be omitted. The value of this member is defined by the Server (e.g. detailed error information, nested errors etc.)." + } } -} \ No newline at end of file + } +} diff --git a/src/utils/Utils.js b/src/utils/Utils.js index ad91a998..3ba1d442 100644 --- a/src/utils/Utils.js +++ b/src/utils/Utils.js @@ -18,7 +18,7 @@ import { CONSTANTS } from '../constant'; import FireboltExampleInvoker from '../FireboltExampleInvoker'; -import errorSchema from '../source/errorSchema.json'; +import errorSchemaObject from '../source/errorSchema.json'; import censorDataJson from 'CensorData'; const { v4: uuidv4 } = require('uuid'); @@ -117,7 +117,6 @@ async function getschemaValidationDone(name, response, sdkType) { * @param response - response of the method */ function censorData(methodName, response) { - let json; try { const json = censorDataJson; if (methodName in json) { @@ -292,11 +291,10 @@ function filterExamples(programlist, programType, offeringType) { return offeringList; } -function errorSchemaCheck(err, communicationMode) { +function errorSchemaCheck(err) { let schemaValidationResult; - if (errorSchema) { - const errorSchemaBasedOnMode = communicationMode == CONSTANTS.TRANSPORT ? errorSchema[CONSTANTS.ERROR_SCHEMA_TRANSPORT] : errorSchema[CONSTANTS.ERROR_SCHEMA_SDK]; - schemaValidationResult = validator.validate(err, errorSchemaBasedOnMode); + if (errorSchemaObject) { + schemaValidationResult = validator.validate(err, errorSchemaObject.errorSchema); } return schemaValidationResult; } diff --git a/test/unit/test_runner.test.js b/test/unit/test_runner.test.js index 0bc92b14..9b531037 100644 --- a/test/unit/test_runner.test.js +++ b/test/unit/test_runner.test.js @@ -29,14 +29,14 @@ const Validator = require('jsonschema').Validator; let MOCK_OPEN_RPC_DOC = { methods: [ { - name: 'account.id', + name: 'Account.id', summary: 'Firebolt OpenRPC schema', params: [], result: { name: 'id', summary: 'the id', schema: { - type: 'object', + type: 'string', }, }, examples: [ @@ -51,7 +51,7 @@ let MOCK_OPEN_RPC_DOC = { ], }, { - name: 'account.uid', + name: 'Account.uid', summary: 'Gets a unique id for the current app & account', params: [], result: { @@ -72,8 +72,328 @@ let MOCK_OPEN_RPC_DOC = { }, ], }, + { + name: 'Device.id', + summary: 'Get the platform back-office device identifier', + params: [], + result: { + name: 'id', + summary: 'the id', + schema: { + type: 'string', + }, + }, + examples: [ + { + name: 'Default Example', + params: [], + result: { + name: 'Default Result', + value: '123', + }, + }, + ], + }, + { + name: 'Device.platform', + summary: 'Get the platform ID for this device', + params: [], + result: { + name: 'platformId', + summary: 'the platform ID', + schema: { + type: 'string', + }, + }, + examples: [ + { + name: 'Getting the platform ID', + params: [], + result: { + name: 'Default Result', + value: 'WPE', + }, + }, + ], + }, + { + name: 'Device.uid', + summary: 'Gets a unique id for the current app & device', + params: [], + result: { + name: 'uniqueId', + summary: 'a unique ID', + schema: { + type: 'string', + }, + }, + examples: [ + { + name: 'Getting the unique ID', + params: [], + result: { + name: 'Default Result', + value: 'ee6723b8-7ab3-462c-8d93-dbf61227998e', + }, + }, + ], + }, + { + name: 'Device.distributor', + summary: 'Get the distributor ID for this device', + params: [], + result: { + name: 'distributorId', + summary: 'the distributor ID', + schema: { + type: 'string', + }, + }, + examples: [ + { + name: 'Getting the distributor ID', + params: [], + result: { + name: 'Default Result', + value: 'Company', + }, + }, + ], + }, + { + name: 'Device.type', + summary: 'Get the device type', + params: [], + result: { + name: 'deviceType', + summary: 'the device type', + schema: { + type: 'string', + }, + }, + examples: [ + { + name: 'Getting the device type', + params: [], + result: { + name: 'Default Result', + value: 'STB', + }, + }, + ], + }, + { + name: 'Device.model', + summary: 'Get the device model', + params: [], + result: { + name: 'model', + summary: 'the device model', + schema: { + type: 'string', + }, + }, + examples: [ + { + name: 'Getting the device model', + params: [], + result: { + name: 'Default Result', + value: 'xi6', + }, + }, + ], + }, + { + name: 'Device.sku', + summary: 'Get the device sku', + params: [], + result: { + name: 'sku', + summary: 'the device sku', + schema: { + type: 'string', + }, + }, + examples: [ + { + name: 'Getting the device sku', + params: [], + result: { + name: 'Default Result', + value: 'AX061AEI', + }, + }, + ], + }, + { + name: 'Device.make', + summary: 'Get the device make', + params: [], + result: { + name: 'make', + summary: 'the device make', + schema: { + type: 'string', + }, + }, + }, + { + name: 'Device.hdcp', + summary: 'Get the supported HDCP profiles', + params: [], + tags: [ + { + name: 'property:readonly', + }, + { + name: 'capabilities', + 'x-uses': ['xrn:firebolt:capability:device:info'], + }, + ], + result: { + name: 'supportedHdcpProfiles', + summary: 'the supported HDCP profiles', + schema: { + type: 'object', + additionalProperties: { + type: 'boolean', + }, + }, + }, + examples: [ + { + name: 'Getting the supported HDCP profiles', + params: [], + result: { + name: 'Default Result', + value: { + 'hdcp1.4': true, + 'hdcp2.2': true, + }, + }, + }, + ], + }, + { + name: 'Accessibility.onClosedCaptionsSettingsChanged', + summary: "Get the user's preferred closed-captions settings", + params: [ + { + name: 'listen', + required: true, + schema: { + type: 'boolean', + }, + }, + ], + tags: [ + { + name: 'subscriber', + 'x-subscriber-for': 'Accessibility.closedCaptionsSettings', + }, + { + name: 'event', + 'x-alternative': 'closedCaptionsSettings', + }, + { + name: 'capabilities', + 'x-uses': ['xrn:firebolt:capability:accessibility:closedcaptions'], + }, + ], + result: { + name: 'closedCaptionsSettings', + summary: 'the closed captions settings', + schema: { + anyOf: [ + { + type: 'object', + required: ['event', 'listening'], + properties: { + event: { + type: 'string', + pattern: '[a-zA-Z]+\\.on[A-Z][a-zA-Z]+', + }, + listening: { + type: 'boolean', + }, + }, + additionalProperties: false, + }, + { + type: 'object', + required: ['enabled', 'styles'], + properties: { + enabled: { + type: 'boolean', + description: 'Whether or not closed-captions should be enabled by default', + }, + styles: { + type: 'object', + description: 'The default styles to use when displaying closed-captions', + }, + preferredLanguages: { + type: 'array', + items: { + type: 'string', + pattern: '^[a-z]{3}$', + }, + }, + }, + }, + ], + }, + }, + examples: [ + { + name: 'Getting the closed captions settings', + params: [ + { + name: 'listen', + value: true, + }, + ], + result: { + name: 'settings', + value: { + enabled: true, + styles: { + fontFamily: 'monospaced_sanserif', + fontSize: 1, + fontColor: '#ffffff', + fontEdge: 'none', + fontEdgeColor: '#7F7F7F', + fontOpacity: 100, + backgroundColor: '#000000', + backgroundOpacity: 100, + textAlign: 'center', + textAlignVertical: 'middle', + windowColor: 'white', + windowOpacity: 50, + }, + preferredLanguages: ['eng', 'spa'], + }, + }, + }, + ], + }, ], }; + +const mockResponses = { + 'Account.id': '123', + 'Account.uid': undefined, + 'Device.platform': { error: 'capability xrn:firebolt:capability:platformn is not supported' }, + 'Device.uid': { error: { code: -32601, message: 'Method not found' } }, + 'Device.distributor': { code: -50100, message: 'capability xrn:firebolt:capability:token:session is not supported' }, + 'Device.type': { error: { message: 'capability xrn:firebolt:capability:Localization.locality is not supported' } }, + 'Device.model': { error: { code: -32601, message: 'capability xrn:firebolt:capability:platformn is not supported' } }, + 'Device.sku': { error: { code: -32601, message: 'Method not found' } }, + 'Device.make': 'Arris', + 'De vice.hdcp': { 'hdcp1.4': true, 'hdcp2.2': true }, + 'Accessibility.onClosedCaptionsSettingsChanged': 'Successful accessibility.listen(closedCaptionsSettingsChanged)', +}; + const EXTERNAL_SDK_MOCK_OPEN_RPC_DOC = { methods: [ { @@ -122,13 +442,7 @@ const EXTERNAL_SDK_MOCK_OPEN_RPC_DOC = { }, ], }; -const MOCK_OPEN_RPC_RESPONSE = { id: 18, result: {}, jsonrpc: '2.0' }; -/** - * This is the definition of the structure used by Validation view - * to create the menu and also to show the result. - * This will be wrapped/transfomed to mocha awesome structure - * if the call is being invoked from Messnger.( Will be changed in FIRECET-72) - */ + const CUSTOM_REPORT_STRUCTURE_SCHEMA = { type: 'object', properties: { @@ -152,83 +466,87 @@ const CUSTOM_REPORT_STRUCTURE_SCHEMA = { }, }; -jest.mock('@apidevtools/json-schema-ref-parser', () => { - return { - dereference: () => { - return new Promise((resolve, reject) => { - if (!mockShouldDereferencerFail) { - resolve(MOCK_OPEN_RPC_DOC); - } else { - reject(new Error('Dereferencer Failure')); - } - }); - }, - }; -}); -/** - * mock object used to emulate the response from - * FireBoltExampleInvoker - */ +jest.mock('@apidevtools/json-schema-ref-parser', () => ({ + dereference: () => + new Promise((resolve, reject) => { + if (!mockShouldDereferencerFail) { + resolve(MOCK_OPEN_RPC_DOC); + } else { + reject(new Error('Dereferencer Failure')); + } + }), +})); + const mockFireboltExampleInvoker = { - invoke: () => {}, + invoke: jest.fn((sdk, methodName, params) => { + return returnMockResponse(methodName); + }), }; -jest.mock('../../src/FireboltExampleInvoker', () => { - return { - get: () => { - return mockFireboltExampleInvoker; - }, - }; -}); -jest.mock('@firebolt-js/sdk/dist/lib/Transport/index.mjs', () => { - return { - send: () => { - return {}; - }, - }; -}); -jest.mock('../../src/FireboltTransportInvoker', () => { - return { - get: () => { - return mockFireboltTransportInvoker; - }, - }; -}); -jest.mock('@firebolt-js/sdk', () => { - return { - Accessibility: {}, - Account: {}, - Advertising: {}, - Authentication: {}, - Device: {}, - Discovery: {}, - Keyboard: {}, - Lifecycle: { - ready: () => {}, - state: () => { - return 'initializing'; // dummy state value. - }, // returning a Lifecycle.state object - close: () => {}, - finish: () => {}, - }, - Localization: {}, - Metrics: {}, - Profile: {}, - Parameters: {}, - SecondScreen: {}, - }; -}); -jest.mock('../../src/pubsub/handlers/RegisterProviderHandler', () => { - return jest.fn().mockImplementation(() => ({ + +const mockFireboltTransportInvoker = { + invoke: jest.fn((methodName, params) => { + return returnMockResponse(methodName); + }), +}; + +function returnMockResponse(methodName) { + return new Promise((resolve, reject) => { + if (mockResponses.hasOwnProperty(methodName)) { + const response = mockResponses[methodName]; + if (response && response.error) { + reject(response.error); + } else { + resolve(response); + } + } else { + resolve({}); + } + }); +} + +jest.mock('../../src/FireboltExampleInvoker', () => ({ + get: () => mockFireboltExampleInvoker, +})); + +jest.mock('@firebolt-js/sdk/dist/lib/Transport/index.mjs', () => ({ + send: jest.fn().mockReturnValue({}), +})); + +jest.mock('../../src/FireboltTransportInvoker', () => ({ + get: () => mockFireboltTransportInvoker, +})); + +jest.mock('@firebolt-js/sdk', () => ({ + Accessibility: {}, + Account: {}, + Advertising: {}, + Authentication: {}, + Device: {}, + Discovery: {}, + Keyboard: {}, + Lifecycle: { + ready: jest.fn(), + state: jest.fn().mockReturnValue('initializing'), + close: jest.fn(), + finish: jest.fn(), + }, + Localization: {}, + Metrics: {}, + Profile: {}, + Parameters: {}, + SecondScreen: {}, +})); + +jest.mock('../../src/pubsub/handlers/RegisterProviderHandler', () => + jest.fn().mockImplementation(() => ({ handle: jest.fn().mockResolvedValue(JSON.stringify({ report: 'registered' })), - })); -}); -jest.mock('../../src/Toast', () => { - const eventEmitter = { - emit: jest.fn(), - }; + })) +); +jest.mock('../../src/Toast', () => { + const eventEmitter = { emit: jest.fn() }; return { - eventEmitter: eventEmitter, + eventEmitter, showToast: (toastMessage, toastState, toastRef) => { eventEmitter.emit('showToast', toastMessage, toastState, toastRef); }, @@ -241,146 +559,216 @@ const mockvalidationViewObj = { jest.mock('../../src/utils/Utils', () => ({ ...jest.requireActual('../../src/utils/Utils'), - pushReportToS3: () => { - return 'restApiUrl'; - }, - censorData: () => { - return 'censoredResponse'; - }, - dereferenceOpenRPC: (mode) => { - if (mode == 'externalsdk') { + pushReportToS3: jest.fn().mockReturnValue('restApiUrl'), + censorData: jest.fn((method, response) => { + return response; + }), + dereferenceOpenRPC: jest.fn((mode) => { + if (mode === 'externalsdk') { return [EXTERNAL_SDK_MOCK_OPEN_RPC_DOC, mode.toLowerCase()]; - } else if (mode == 'core' || mode == 'manage') { + } else if (mode === 'core' || mode === 'manage') { return [MOCK_OPEN_RPC_DOC, mode.toLowerCase()]; } - }, + }), })); -jest.mock('lodash', () => { - return { - cloneDeep: (value) => { - return value; - }, - }; -}); +jest.mock('lodash', () => ({ + cloneDeep: jest.fn((value) => value), +})); let mockShouldDereferencerFail = false; let runner; let result; const navigation = ''; +const INCLUDE_EVENT_METHODS = []; + +jest.mock('../../src/MethodFilters', () => ({ + __esModule: true, + default: jest.fn().mockImplementation(() => ({ + isExceptionMethod: jest.fn((methodName) => { + const exceptionMethods = ['Device.distributor', 'Device.type', 'Device.model', 'Device.sku']; + return exceptionMethods.includes(methodName); + }), + isMethodToBeExcluded: jest.fn((methodObject) => { + const excludedMethodsList = ['Device.hdcp']; + return excludedMethodsList.includes(methodObject.name); + }), + isRpcMethod: jest.fn(), + isSubscribeMethod: jest.fn(), + isSetMethod: jest.fn(), + shouldExcludeExample: jest.fn(), + isEventMethod: jest.fn((method) => { + let isEvent = false; + if (method.tags && INCLUDE_EVENT_METHODS.indexOf(method.name) === -1) { + method.tags.forEach((tag) => { + if (tag.name && tag.name === 'event') { + isEvent = true; + } + }); + } + return isEvent; + }), + })), +})); describe('Test_Runner test cases', () => { beforeEach(() => { runner = new Test_Runner(); - (runner.reportGenenration = jest.fn().mockImplementationOnce(() => { - return new Promise((resolve) => { - resolve(''); - }); - })), - (runner.invokeLifecycleAPI = jest.fn().mockImplementationOnce((tempParams) => { - if (tempParams.methodName == CONSTANTS.LIFECYCLE_METHOD_LIST[1]) { - return 'initializing'; - } else { - const mockLifecycleHistoryget = { _history: [{ someKey: 'someValue' }] }; - return mockLifecycleHistoryget; - } - })); + runner.reportGenenration = jest.fn().mockResolvedValue(''); + runner.invokeLifecycleAPI = jest.fn().mockImplementation((tempParams) => { + if (tempParams.methodName === CONSTANTS.LIFECYCLE_METHOD_LIST[1]) { + return 'initializing'; + } else { + return { _history: [{ someKey: 'someValue' }] }; + } + }); }); + describe('northBoundSchemaValidationAndReportGeneration Scenarios', () => { - test('Validate northBoundSchemaValidationAndReportGeneration(SDK) when OPEN RPC dereferece call fails', async () => { + test('should return empty result when dereference call fails for SDK', async () => { mockShouldDereferencerFail = true; - result = await runner.northBoundSchemaValidationAndReportGeneration('SDK', navigation, mockvalidationViewObj); + result = await runner.northBoundSchemaValidationAndReportGeneration('SDK'); /** when the dereference fails it should not execute any api and the result list will have 0 elements */ expect(result.length).toEqual(0); }); - test('Validate northBoundSchemaValidationAndReportGeneration(CORE) with example with valid response from FireboltExapmpleInvoker', async () => { - mockShouldDereferencerFail = false; - // Mock a valid response coming back from the Firebolt Invoker - mockFireboltExampleInvoker.invoke = () => Promise.resolve(MOCK_OPEN_RPC_RESPONSE); - result = await runner.northBoundSchemaValidationAndReportGeneration(CONSTANTS.CORE); - /** - * Since the mocked OPEN_RPC has 2 documents we will have 2 results - */ - expect(result.length).toEqual(2); - const v = new Validator(); - const schemaMapResult = v.validate(result[0], CUSTOM_REPORT_STRUCTURE_SCHEMA); - // This would make sure that the result json that is created is in valid strcuture - // we are not bothered about the value in the json. - expect(schemaMapResult.errors.length).toEqual(0); - /** - * Validate if the response title is populated correctly - */ - expect(result[1].fullTitle).toEqual('account.uid.Getting the unique ID'); - expect(result[0].state).toEqual('passed'); - }); - test('Validate northBoundSchemaValidationAndReportGeneration(CORE) with a invalid schema response coming back from FireboltExampleInvoker', async () => { - mockShouldDereferencerFail = false; - // Mock an invalid schema response coming back from the Firebolt Invoker - mockFireboltExampleInvoker.invoke = () => Promise.resolve(null); + describe('northBoundSchemaValidationAndReportGeneration Scenarios for CORE', () => { + beforeAll(async () => { + result = await runner.northBoundSchemaValidationAndReportGeneration(CONSTANTS.CORE); + }); + test('should return valid response for Account.id API', async () => { + const extractedResult = result.find((obj) => obj.title === 'Account.id'); + extractedResult.code = JSON.parse(extractedResult.code); + expect(extractedResult.code['Schema Validation']).toEqual('Passed'); + expect(extractedResult.code.Response.result).toBeDefined(); + }); - result = await runner.northBoundSchemaValidationAndReportGeneration(CONSTANTS.CORE); - /** - * Since the mocked OPEN_RPC has 2 documents we will have 2 results - */ - expect(result.length).toEqual(2); - // Both results should have resulted in schema validation failure - expect(result[0].state).toEqual('failed'); + test('should fail for Account.uid API due to undefined response', async () => { + const extractedResult = result.find((obj) => obj.title === 'Account.uid'); + extractedResult.code = JSON.parse(extractedResult.code); + expect(extractedResult.code['Schema Validation']).toEqual('Failed'); + expect(extractedResult.code.Response).toBeNull(); + expect(extractedResult.code.Message).toContain('No result or error in response.'); + }); + + test('should handle different type of response for Device.id API', async () => { + const extractedResult = result.find((obj) => obj.title === 'Device.id'); + extractedResult.code = JSON.parse(extractedResult.code); + expect(extractedResult.code['Schema Validation']).toEqual('Failed'); + expect(extractedResult.code.Response.result).toBeDefined(); + expect(extractedResult.code.Message).toContain('instance.result is not of a type(s) string'); + }); + + test('should handle unexpected error for Device.platform API', async () => { + const extractedResult = result.find((obj) => obj.title === 'Device.platform'); + extractedResult.code = JSON.parse(extractedResult.code); + expect(extractedResult.code['Schema Validation']).toEqual('Failed'); + expect(extractedResult.code.Response.error).toBeDefined(); + expect(extractedResult.code.Message).toContain('Unexpected error encountered in the response'); + }); + + test('should handle unexpected error: "method not implemented" for Device.uid API', async () => { + const extractedResult = result.find((obj) => obj.title === 'Device.uid'); + extractedResult.code = JSON.parse(extractedResult.code); + expect(extractedResult.code['Schema Validation']).toEqual('Failed'); + expect(extractedResult.code.Response.error).toBeDefined(); + expect(extractedResult.code.Message).toContain('Method not implemented by platform'); + }); + + test('should handle expecting error but received result for Device.distributor API', async () => { + const extractedResult = result.find((obj) => obj.title === 'Device.distributor'); + extractedResult.code = JSON.parse(extractedResult.code); + expect(extractedResult.code['Schema Validation']).toEqual('Failed'); + expect(extractedResult.code.Response.result).toBeDefined(); + expect(extractedResult.code.Message).toContain('Expected error, received result'); + }); + + test('should handle expecting error but incorrect error format for Device.type API', async () => { + const extractedResult = result.find((obj) => obj.title === 'Device.type'); + extractedResult.code = JSON.parse(extractedResult.code); + expect(extractedResult.code['Schema Validation']).toEqual('Failed'); + expect(extractedResult.code.Response.error).toBeDefined(); + expect(extractedResult.code.Message).toContain('Expected error, incorrect error format'); + }); + + test('should handle expected error but received error for Device.model API', async () => { + const extractedResult = result.find((obj) => obj.title === 'Device.model'); + extractedResult.code = JSON.parse(extractedResult.code); + expect(extractedResult.code['Schema Validation']).toEqual('Passed'); + expect(extractedResult.code.Response.error).toBeDefined(); + expect(extractedResult.code.Message).toContain('Expected error, received error'); + }); + + test('should handle method not implemented error for Device.sku API', async () => { + const extractedResult = result.find((obj) => obj.title === 'Device.sku'); + extractedResult.code = JSON.parse(extractedResult.code); + expect(extractedResult.code['Schema Validation']).toEqual('Passed'); + expect(extractedResult.code.Response.error).toBeDefined(); + expect(extractedResult.code.Message).toContain('Method not implemented by platform'); + }); + + test('should skip validation for Device.make API due to missing example', async () => { + const extractedResult = result.find((obj) => obj.title === 'Device.make'); + extractedResult.code = JSON.parse(extractedResult.code); + expect(extractedResult.code['Schema Validation']).toEqual('Skipped'); + expect(extractedResult.code.Response).toBeNull(); + expect(extractedResult.code.Message).toContain('Could not find an example for Device.make'); + }); + + test('should return valid response for Accessibility.onClosedCaptionsSettingsChanged Event', async () => { + const extractedResult = result.find((obj) => obj.title === 'Accessibility.onClosedCaptionsSettingsChanged'); + extractedResult.code = JSON.parse(extractedResult.code); + expect(extractedResult.code['Schema Validation']).toEqual('Passed'); + expect(extractedResult.code.Response.result).toBeDefined(); + }); }); - test('Validate northBoundSchemaValidationAndReportGeneration when invalid mode is passed', async () => { - /** when the invalid mode is passed dereference fails it should not execute any api */ + + test('should return error when invalid mode is passed', async () => { mockShouldDereferencerFail = false; result = await runner.northBoundSchemaValidationAndReportGeneration('undefined', navigation, mockvalidationViewObj); expect(result.error).toEqual(CONSTANTS.NOTPERFORMED); }); - test('Validate northBoundSchemaValidationAndReportGeneration(CORE) with no example with valid response from FireboltExapmpleInvoker', async () => { - mockShouldDereferencerFail = false; - // Mock a valid response coming back from the Firebolt Invoker - // Overriding MOCK_OPEN_RPC_DOC value for device module test - + test('should handle when mode as passes as an arry', async () => { MOCK_OPEN_RPC_DOC = { methods: [ { - name: 'device.uid', + name: 'Account.id', summary: 'Firebolt OpenRPC schema', params: [], result: { - name: 'OpenRPC Schema', + name: 'id', + summary: 'the id', schema: { - type: 'object', + type: 'string', }, }, + examples: [ + { + name: 'Default Example', + params: [], + result: { + name: 'Default Result', + value: '123', + }, + }, + ], }, ], }; - mockFireboltExampleInvoker.invoke = () => Promise.resolve(MOCK_OPEN_RPC_RESPONSE); - - result = await runner.northBoundSchemaValidationAndReportGeneration(CONSTANTS.CORE); - /** - * Since the mocked OPEN_RPC has 2 documents we will have 2 results - */ - expect(result.length).toEqual(1); - const v = new Validator(); - const schemaMapResult = v.validate(result[0], CUSTOM_REPORT_STRUCTURE_SCHEMA); - // This would make sure that the result json that is created is in valid strcuture - // we are not bothered about the value in the json. - expect(schemaMapResult.errors.length).toEqual(0); - /** - * Validate if the response title is populated correctly - */ - expect(result[0].fullTitle).toEqual('device.uid'); - expect(result[0].code).toContain('Could not find an example for device.uid'); + result = await runner.northBoundSchemaValidationAndReportGeneration([CONSTANTS.CORE]); + const extractedResult = result.find((obj) => obj.title === 'Account.id'); + extractedResult.code = JSON.parse(extractedResult.code); + expect(extractedResult.code['Schema Validation']).toEqual('Passed'); + expect(extractedResult.code.Response.result).toBeDefined(); }); - test('Validate northBoundSchemaValidationAndReportGeneration when module is device (schema and content validation from library)', async () => { - /** when the module is device, content and schema validation would be done externally */ - mockShouldDereferencerFail = false; - // Overriding MOCK_OPEN_RPC_DOC value for device module test + + test('should handle when communication mode is Transport', async () => { MOCK_OPEN_RPC_DOC = { methods: [ { - name: 'device.id', - summary: 'Get the platform back-office device identifier', + name: 'Account.id', + summary: 'Firebolt OpenRPC schema', params: [], result: { name: 'id', @@ -402,43 +790,45 @@ describe('Test_Runner test cases', () => { }, ], }; - mockFireboltExampleInvoker.invoke = () => Promise.resolve(MOCK_OPEN_RPC_RESPONSE); - result = await runner.northBoundSchemaValidationAndReportGeneration(CONSTANTS.CORE, navigation, mockvalidationViewObj); - const parsedCode = JSON.parse(result[0].code); - // Schema validation is expected to fail as MOCK_OPEN_RPC_RESPONSE is not in the expected schema for device module - expect(parsedCode.Schema).toEqual('Failed'); - // Content validation will be skipped when schema validation fails - expect(parsedCode.Content).toEqual('Pending'); - - // Reverting MOCK_OPEN_RPC_DOC to previous value + process.env.COMMUNICATION_MODE = 'Transport'; + result = await runner.northBoundSchemaValidationAndReportGeneration([CONSTANTS.CORE]); + const extractedResult = result.find((obj) => obj.title === 'Account.id'); + extractedResult.code = JSON.parse(extractedResult.code); + expect(extractedResult.code['Schema Validation']).toEqual('Passed'); + expect(extractedResult.code.Response.result).toBeDefined(); + }); + + test('should handle when schema is missing from openRpc', async () => { MOCK_OPEN_RPC_DOC = { methods: [ { - name: 'rpc.discover', + name: 'Account.id', summary: 'Firebolt OpenRPC schema', params: [], result: { - name: 'OpenRPC Schema', - schema: { - type: 'object', - }, + name: 'id', + summary: 'the id', }, - }, - { - name: 'account.id', - summary: 'Firebolt OpenRPC schema', - params: [], - result: { - name: 'OpenRPC Schema', - schema: { - type: 'object', + examples: [ + { + name: 'Default Example', + params: [], + result: { + name: 'Default Result', + value: '123', + }, }, - }, + ], }, ], }; + result = await runner.northBoundSchemaValidationAndReportGeneration([CONSTANTS.CORE]); + const extractedResult = result.find((obj) => obj.title === 'Account.id'); + extractedResult.code = JSON.parse(extractedResult.code); + expect(extractedResult.code['Schema Validation']).toEqual('Skipped'); }); }); + describe('UUID Generation Validation', () => { let firstuuid, seconduuid; test('validate uuid when generated two uuid are different', () => { @@ -520,7 +910,7 @@ describe('Test_Runner test cases', () => { test('Validate northBoundSchemaValidationAndReportGeneration(externalSDK) with example with valid response from FireboltExapmpleInvoker', async () => { mockShouldDereferencerFail = false; // Mock a valid response coming back from the Firebolt Invoker - mockFireboltExampleInvoker.invoke = () => Promise.resolve(MOCK_OPEN_RPC_RESPONSE); + mockFireboltExampleInvoker.invoke = () => Promise.resolve({ id: 18, result: {}, jsonrpc: '2.0' }); result = await runner.northBoundSchemaValidationAndReportGeneration('externalSDK'); /**