From 359e356e338f015bffb7854a9eae696cd93abec4 Mon Sep 17 00:00:00 2001 From: SPANIER Nathan Date: Tue, 20 Apr 2021 17:04:06 +0200 Subject: [PATCH] feature(gen-test-payload): ivs and livestream are working correctly (missing vod), scripts are generated from helpers files then used in scripts/setup.js which is invoked on 'npm run test', also modified 'name' to 'resourceName' in livestream questions --- .../service-walkthroughs/livestream-push.js | 2 +- .../awscloudformation/utils/video-staging.js | 2 +- provider-utils/livestream-questions.json | 2 +- provider-utils/supported-services.json | 53 +++--- scripts/gen-test-payload/index.js | 167 ++++++++++++------ .../output/{.gitinclude => .gitkeep} | 0 scripts/gen-test-payload/template.ejs | 18 +- .../test-helpers/ivs-helpers.json | 2 +- .../test-helpers/livestream-helpers.json | 67 ++++--- .../test-helpers/video-on-demand-helpers.json | 131 ++++++++++++++ .../test-helpers/vod-helpers.json | 1 - scripts/headless/add-vod.sh | 2 +- scripts/setup.js | 13 +- 13 files changed, 341 insertions(+), 119 deletions(-) rename scripts/gen-test-payload/output/{.gitinclude => .gitkeep} (100%) create mode 100644 scripts/gen-test-payload/test-helpers/video-on-demand-helpers.json delete mode 100644 scripts/gen-test-payload/test-helpers/vod-helpers.json diff --git a/provider-utils/awscloudformation/service-walkthroughs/livestream-push.js b/provider-utils/awscloudformation/service-walkthroughs/livestream-push.js index a3a8648a..ae15e756 100644 --- a/provider-utils/awscloudformation/service-walkthroughs/livestream-push.js +++ b/provider-utils/awscloudformation/service-walkthroughs/livestream-push.js @@ -323,7 +323,7 @@ async function serviceQuestions(context, options, defaultValuesFilename, resourc // main question control flow const answers = {}; - answers.resourceName = resource.name; + answers.resourceName = resource.resourceName; const advancedEnable = await inquirer.prompt(advanced); if (advancedEnable.advancedChoice === false) { diff --git a/provider-utils/awscloudformation/utils/video-staging.js b/provider-utils/awscloudformation/utils/video-staging.js index 1a3c639f..5b8cccf0 100644 --- a/provider-utils/awscloudformation/utils/video-staging.js +++ b/provider-utils/awscloudformation/utils/video-staging.js @@ -10,7 +10,7 @@ const { getAWSConfig } = require('./get-aws'); async function buildTemplates(context, props) { const { amplify } = context; - const amplifyMeta = amplify.getProjectMeta(); + const amplifyMeta = await amplify.getProjectMeta(); const { serviceType } = amplifyMeta.video[props.shared.resourceName]; const spinner = ora('Building video resources...'); spinner.start(); diff --git a/provider-utils/livestream-questions.json b/provider-utils/livestream-questions.json index 7368ef7b..19c37261 100644 --- a/provider-utils/livestream-questions.json +++ b/provider-utils/livestream-questions.json @@ -2,7 +2,7 @@ "video": { "inputs": [ { - "key": "name", + "key": "resourceName", "question": "Provide a friendly name for your resource to be used as a label for this category in the project:", "validation": { "operator": "regex", diff --git a/provider-utils/supported-services.json b/provider-utils/supported-services.json index 44672ae0..116537fb 100644 --- a/provider-utils/supported-services.json +++ b/provider-utils/supported-services.json @@ -1,26 +1,29 @@ { - "livestream":{ - "alias":"AWS Elemental Live Stream", - "serviceWalkthroughFilename":"livestream-push.js", - "cfnFilename":"livestream-workflow-template.json.ejs", - "stackFolder":"livestream-helpers", - "defaultValuesFilename":"livestream-defaults.json", - "provider":"awscloudformation" - }, - "ivs":{ - "alias":"Amazon Interactive Video Service Live Stream (Beta)", - "serviceWalkthroughFilename":"ivs-push.js", - "cfnFilename":"ivs-workflow-template.yaml.ejs", - "stackFolder":"ivs-helpers", - "defaultValuesFilename":"ivs-defaults.json", - "provider":"awscloudformation" -}, - "video-on-demand":{ - "alias":"Video-On-Demand", - "serviceWalkthroughFilename":"vod-push.js", - "cfnFilename":"vod-workflow-template.yaml.ejs", - "stackFolder":"vod-helpers", - "defaultValuesFilename":"vod-defaults.json", - "provider":"awscloudformation" - } -} \ No newline at end of file + "livestream": { + "alias": "AWS Elemental Live Stream", + "serviceWalkthroughFilename": "livestream-push.js", + "cfnFilename": "livestream-workflow-template.json.ejs", + "stackFolder": "livestream-helpers", + "defaultValuesFilename": "livestream-defaults.json", + "provider": "awscloudformation", + "questionFilename": "livestream-questions.json" + }, + "ivs": { + "alias": "Amazon Interactive Video Service Live Stream (Beta)", + "serviceWalkthroughFilename": "ivs-push.js", + "cfnFilename": "ivs-workflow-template.yaml.ejs", + "stackFolder": "ivs-helpers", + "defaultValuesFilename": "ivs-defaults.json", + "provider": "awscloudformation", + "questionFilename": "ivs-questions.json" + }, + "video-on-demand": { + "alias": "Video-On-Demand", + "serviceWalkthroughFilename": "vod-push.js", + "cfnFilename": "vod-workflow-template.yaml.ejs", + "stackFolder": "vod-helpers", + "defaultValuesFilename": "vod-defaults.json", + "provider": "awscloudformation", + "questionFilename": "vod-questions.json" + } +} diff --git a/scripts/gen-test-payload/index.js b/scripts/gen-test-payload/index.js index 0cb098dd..97802637 100644 --- a/scripts/gen-test-payload/index.js +++ b/scripts/gen-test-payload/index.js @@ -1,24 +1,19 @@ const fs = require('fs'); const ejs = require('ejs'); +let fullCoverage = false; + const servicesQuestions = []; const servicesHelpers = {}; -const questionFolder = `${__dirname}/../../provider-utils/`; -fs.readdirSync(questionFolder).forEach((file) => { - if (file.includes('questions')) { - servicesQuestions.push({ - serviceType: file.split('-')[0], - content: require(`${questionFolder}${file}`), - }); - } -}); - -const helpersFolder = './test-helpers/'; -fs.readdirSync(helpersFolder).forEach((file) => { - if (file.includes('helpers')) { - servicesHelpers[file.split('-')[0]] = require(`${helpersFolder}${file}`); - } +const supportedServices = require(`${__dirname}/../../provider-utils/supported-services.json`); +const helpersFolder = `${__dirname}/test-helpers/`; +Object.keys(supportedServices).forEach((key) => { + servicesQuestions.push({ + serviceType: `${key}`, + provider: `${supportedServices[key].questionFilename}`, + }); + servicesHelpers[key] = require(`${helpersFolder}${key}-helpers.json`); }); class TreeNode { @@ -38,85 +33,129 @@ class TreeNode { } class Tree { - constructor() { + constructor(serviceType, provider) { + this.serviceType = serviceType; + this.provider = provider; this.maxDepth = 0; this.paths = []; } - buildTree(questions, helper) { + buildTree(helper) { const getNextNode = (currNode, currQuestion, depth) => { depth++; - if (typeof currQuestion === 'undefined' || (!!currQuestion.next && !!currQuestion.options) || (typeof currQuestion.ignore !== 'undefined')) { + this.maxDepth = depth; + if (typeof currQuestion === 'undefined' + || (!!currQuestion.next && !!currQuestion.options && currQuestion.type !== 'checkbox')) { return; } if (!!currQuestion.type && currQuestion.type === 'list') { Object.keys(currQuestion.options).forEach((optionKey) => { - if (currQuestion.options[optionKey].ignore === true) { + if (currQuestion.options[optionKey].ignore === true && !fullCoverage) { return; } if (!currQuestion.options[optionKey].next) { - currNode.addChild(new TreeNode(currQuestion.options[optionKey].value, + currNode.addChild(new TreeNode(currQuestion.options[optionKey].value || '', currQuestion.key, depth)); } else { - const nextNode = new TreeNode(currQuestion.options[optionKey].value, - currQuestion.key, depth); - currNode.addChild(nextNode); - getNextNode(nextNode, helper[currQuestion.options[optionKey].next], - depth); + currQuestion.options[optionKey].next.split('||').forEach((next) => { + const nextNode = new TreeNode(currQuestion.options[optionKey].value || '', + currQuestion.key, depth); + currNode.addChild(nextNode); + getNextNode(nextNode, helper[next], + depth); + }); } }); } else if (!!currQuestion.type && currQuestion.type === 'confirm') { - // don't create node out of those because they don't hold payload - // they might hold one but we have no example for that - getNextNode(currNode, helper[currQuestion.yesNext], depth); // yes - getNextNode(currNode, helper[currQuestion.noNext], depth); // no + currQuestion.options.forEach((option) => { + if (option.ignore === true && !fullCoverage) { + return; + } + if (!option.next) { + const nextNode = new TreeNode(option.value, currQuestion.key, depth); + currNode.addChild(nextNode); + } else { + option.next.split('||').forEach((next) => { + const nextNode = new TreeNode(option.value, currQuestion.key, depth); + currNode.addChild(nextNode); + getNextNode(nextNode, helper[next], depth); + }); + } + }); } else if (!!currQuestion.type && currQuestion.type === 'checkbox') { - // gen all permutations of checkboxes and call getNextNode for each of them + const getCombinations = (array) => { + const result = []; + const fGetCombinations = (prefix, arr) => { + for (let i = 0; i < arr.length; i++) { + result.push(`${prefix},${arr[i]}`); + fGetCombinations(`${prefix},${arr[i]}`, arr.slice(i + 1)); + } + }; + fGetCombinations('', array); + return result.map(r => r.slice(1, r.length)).map(r => r.split(',')); + }; + getCombinations(currQuestion.options).forEach((combinaison) => { + const nextNode = new TreeNode(combinaison || '', + currQuestion.key, depth); + currNode.addChild(nextNode); + getNextNode(nextNode, helper[currQuestion.next], depth); + }); + } else if (currQuestion.next) { + currQuestion.next.split('||').forEach((next) => { + const nextNode = new TreeNode(helper[currQuestion.key].value || '', + currQuestion.key, depth); + currNode.addChild(nextNode); + getNextNode(nextNode, helper[next], depth); + }); } else { - const nextNode = new TreeNode(helper[currQuestion.key].defaultValue, - currQuestion.key, depth); - currNode.addChild(nextNode); - getNextNode(nextNode, helper[currQuestion.next], depth); + currNode.addChild(new TreeNode(helper[currQuestion.key].value || '', + currQuestion.key, depth)); } }; this.rootNode = new TreeNode('root', 'root', 0); - const firstQuestion = helper[questions.content.video.inputs[0].key]; - const firstNode = new TreeNode(helper[questions.content.video.inputs[0].key].defaultValue, - questions.content.video.inputs[0].key, 1); + const firstQuestion = helper.resourceName; + const firstNode = new TreeNode(firstQuestion.value, 'resourceName', 1); this.rootNode.addChild(firstNode); getNextNode(firstNode, helper[firstQuestion.next], firstNode.depth); return this; } - buildPaths(node, path, helpers) { - if (node.childs.length === 0) { - path = [...path, { key: node.key, value: node.value }]; - this.paths = [...this.paths, path]; + buildPaths(node, path) { + if (node.parents.length === 0) { + this.paths = [...this.paths, [...path]]; return; } if (node.key !== 'root') { path = [...path, { key: node.key, value: node.value }]; } - node.childs.forEach((c) => { - this.buildPaths(c, path, helpers); + node.parents.forEach((c) => { + this.buildPaths(c, [...path]); }); } - buildScript(questions) { + getBottomNodes(node, arr) { + if (node.childs.length === 0) { + arr.push(node); + } else { + node.childs.forEach(child => this.getBottomNodes(child, arr)); + } + } + + buildScript() { this.paths.forEach((path, idx) => { - ejs.renderFile('./template.ejs', { + ejs.renderFile(`${__dirname}/template.ejs`, { payload: { inputs: path, - serviceType: questions.serviceType, - provider: questions.content.video.provider, + serviceType: this.serviceType, + provider: this.provider, }, }, (ejsErr, str) => { if (ejsErr) { console.error(ejsErr); return; } - fs.writeFile(`output/${questions.serviceType}-${idx}.sh`, str, (fsErr) => { + fs.writeFile(`${__dirname}/output/${this.serviceType}-${idx}.sh`, str, (fsErr) => { if (fsErr) { console.error(fsErr); } @@ -126,14 +165,30 @@ class Tree { } } -// Entrypoint (node index.js) +if (process.argv.length !== 3 && process.argv.length !== 2) { + console.info("Arguments should be 'node index.js [full-coverage]"); + process.exit(1); +} + +fullCoverage = process.argv[2] === 'full-coverage'; + +console.info('\nGenerating script files...'); + servicesQuestions.forEach((question) => { - if (question.serviceType === 'livestream' || question.serviceType === 'ivs') { - console.info(`---Service ${question.serviceType}---`); - const tree = new Tree(); - tree.buildTree(question, servicesHelpers[`${question.serviceType}`]); - tree.buildPaths(tree.rootNode, [], servicesHelpers[`${question.serviceType}`].content); - console.info('Number of permutations:', tree.paths.length); - tree.buildScript(question); + if (['livestream', 'ivs', 'video-on-demand'].includes(question.serviceType)) { + const tree = new Tree(question.serviceType, question.provider); + tree.buildTree(servicesHelpers[`${question.serviceType}`]); + const bottomNodes = []; + tree.getBottomNodes(tree.rootNode, bottomNodes); + bottomNodes.forEach(node => tree.buildPaths(node, [])); + tree.paths.forEach((path) => { + path.forEach((elem) => { + if (elem.key === 'resourceName') { + elem.value += `${Math.random().toString(36).substring(2, 6)}`; + } + }); + }); + tree.buildScript(); + console.info(`Generated ${tree.paths.length} scripts for ${question.serviceType} service`); } }); diff --git a/scripts/gen-test-payload/output/.gitinclude b/scripts/gen-test-payload/output/.gitkeep similarity index 100% rename from scripts/gen-test-payload/output/.gitinclude rename to scripts/gen-test-payload/output/.gitkeep diff --git a/scripts/gen-test-payload/template.ejs b/scripts/gen-test-payload/template.ejs index 6812239b..2cfff4c0 100644 --- a/scripts/gen-test-payload/template.ejs +++ b/scripts/gen-test-payload/template.ejs @@ -2,12 +2,18 @@ set -e IFS='|' -PERMUTATION="{ -\"service\":\"video\", -\"serviceType\":\"<%- payload.serviceType %>\", -\"providerName\":\"<%- payload.provider %>\", -<% payload.inputs.forEach(({key, value}) => { %> -\"<%- key %>\":\"<%- value %>\", +PERMUTATION="{\ +\"service\":\"video\",\ +\"serviceType\":\"<%- payload.serviceType %>\",\ +\"providerName\":\"<%- payload.provider %>\",\ +<% let token = ','; %> +<% payload.inputs.forEach(({key, value}, index, array) => { %> +<% if (index === array.length - 1) { token = '' } %> +<% if (value === 'true' || value === 'false') { %> +\"<%- key %>\":<%- value %><%- token %>\ +<% } else { %> +\"<%- key %>\":\"<%- value %>\"<%- token %>\ +<% } %> <% }); %> }" amplify video add --payload $PERMUTATION diff --git a/scripts/gen-test-payload/test-helpers/ivs-helpers.json b/scripts/gen-test-payload/test-helpers/ivs-helpers.json index ce014ee1..e08c6ced 100644 --- a/scripts/gen-test-payload/test-helpers/ivs-helpers.json +++ b/scripts/gen-test-payload/test-helpers/ivs-helpers.json @@ -2,7 +2,7 @@ "resourceName": { "key": "ressourceName", "next": "channelQuality", - "defaultValue": "mylivestream" + "value": "mylivestream" }, "channelQuality": { "key": "channelQuality", diff --git a/scripts/gen-test-payload/test-helpers/livestream-helpers.json b/scripts/gen-test-payload/test-helpers/livestream-helpers.json index 266361c0..386b5ea6 100644 --- a/scripts/gen-test-payload/test-helpers/livestream-helpers.json +++ b/scripts/gen-test-payload/test-helpers/livestream-helpers.json @@ -1,28 +1,38 @@ { - "name": { - "key": "name", - "next": "gopSize", - "defaultValue": "mylivestream" + "resourceName": { + "key": "resourceName", + "next": "advancedChoice", + "value": "mylivestream" + }, + "advancedChoice": { + "key": "advancedChoice", + "type": "confirm", + "options": [ + { + "value": "true", + "next": "gopSize" + } + ] }, "gopSize": { "key": "gopSize", "next": "gopPerSegment", - "defaultValue": "1" + "value": "1" }, "gopPerSegment": { "key": "gopPerSegment", "next": "segsPerPlist", - "defaultValue": "2" + "value": "2" }, "segsPerPlist": { "key": "segsPerPlist", "next": "securityGroup", - "defaultValue": "3" + "value": "3" }, "securityGroup": { "key": "securityGroup", "next": "encodingProfile", - "defaultValue": "0.0.0.0/0" + "value": "0.0.0.0/0" }, "ingestType": { "key": "ingestType", @@ -30,12 +40,12 @@ "options": { "RTMP_PUSH": { "next": "storageType", - "value": "RTMP_PUSH", - "ignore": true + "value": "RTMP_PUSH" }, "RTP_PUSH": { "next": "storageType", - "value": "RTP_PUSH" + "value": "RTP_PUSH", + "ignore": true }, "UDP_PUSH": { "next": "storageType", @@ -55,7 +65,8 @@ "options": { "UHD": { "next": "autoStart", - "value": "UHD" + "value": "UHD", + "ignore": true }, "FULL": { "next": "autoStart", @@ -63,15 +74,18 @@ }, "MOBILE": { "next": "autoStart", - "value": "MOBILE" + "value": "MOBILE", + "ignore": true }, "HD": { "next": "autoStart", - "value": "HD" + "value": "HD", + "ignore": true }, "SD": { "next": "autoStart", - "value": "SD" + "value": "SD", + "ignore": true } } }, @@ -85,7 +99,8 @@ }, "NO": { "next": "ingestType", - "value": "NO" + "value": "NO", + "ignore": true } } }, @@ -142,7 +157,7 @@ "startOverWindow": { "key": "startOverWindow", "next": "enableDistribution", - "defaultValue": "86400" + "value": "86400" }, "enableMediaStore": { "key": "enableMediaStore", @@ -177,11 +192,13 @@ "options": { "PriceClass_100": { "next": "sBucketLogs", - "value": "PriceClass_100" + "value": "PriceClass_100", + "ignore": true }, "PriceClass_200": { "next": "sBucketLogs", - "value": "PriceClass_200" + "value": "PriceClass_200", + "ignore": true }, "PriceClass_All": { "next": "sBucketLogs", @@ -192,11 +209,11 @@ "sBucketLogs": { "key": "sBucketLogs", "next": "sLogPrefix", - "defaultValue": "" + "value": "s3://amplifyvideotest/logs/" }, "sLogPrefix": { "key": "sLogPrefix", - "defaultValue": "cf_logs/" + "value": "cf_logs/" }, "storageType": { "key": "storageType", @@ -204,11 +221,13 @@ "options": { "mPackage": { "next": "endpoints", - "value": "mPackage" + "value": "mPackage", + "ignore": true }, "mPackageStore": { "next": "endpoints", - "value": "mPackageStore" + "value": "mPackageStore", + "ignore": true }, "mStore": { "next": "enableDistribution", @@ -219,6 +238,6 @@ "mp4URL": { "key": "mp4URL", "next": "storageType", - "defaultValue": "" + "value": "s3://amplifyvideotest/VANLIFE.mp4" } } diff --git a/scripts/gen-test-payload/test-helpers/video-on-demand-helpers.json b/scripts/gen-test-payload/test-helpers/video-on-demand-helpers.json new file mode 100644 index 00000000..44d377c3 --- /dev/null +++ b/scripts/gen-test-payload/test-helpers/video-on-demand-helpers.json @@ -0,0 +1,131 @@ +{ + "resourceName": { + "key": "resourceName", + "next": "enableCDN", + "value": "myvodservice" + }, + "encodingTemplate": { + "key": "encodingTemplate", + "type": "list", + "options": { + "System-Ott_Hls_Ts_Avc_Aac": { + "value": "System-Ott_Hls_Ts_Avc_Aac", + "next": "enableCDN" + }, + "advanced": { + "next": "encodingTemplate-advanced", + "ignore": true + } + } + }, + "encodingTemplate-advanced": { + "key": "encodingTemplate-advanced", + "next": "enableCDN" + }, + "enableCDN": { + "key": "enableCDN", + "type": "confirm", + "options": [ + { + "value": "true", + "next": "modifySignedUrl", + "ignore": true + }, + { + "value": "true", + "next": "signedKey" + }, + { + "value": "false", + "next": "enableCMS" + } + ] + }, + "enableCMS": { + "key": "enableCMS", + "type": "confirm", + "options": [ + { + "value": "true", + "next": "permissionSchema", + "ignore": true + }, + { + "value": "false" + } + ] + }, + "editAPI": { + "key": "editAPI", + "type": "confirm", + "options": [ + { + "value": "true", + "ignore": true + }, + { + "value": "false", + "ignore": true + } + ] + }, + "modifySignedUrl": { + "key": "modifySignedUrl", + "type": "list", + "options": { + "leave": { + "value": "leave", + "next": "enableCMS", + "ignore": true + }, + "rotate": { + "value": "rotate", + "next": "enableCMS", + "ignore": true + }, + "remove": { + "value": "remove", + "next": "enableCMS", + "ignore": true + } + } + }, + "signedKey": { + "key": "signedKey", + "type": "confirm", + "options": [ + { + "value": "true", + "next": "enableCMS" + }, + { + "value": "false", + "next": "enableCMS", + "ignore": true + } + ] + }, + "overrideSchema": { + "key": "overrideSchema", + "type": "confirm", + "options": [ + { + "value": "true", + "next": "editAPI", + "ignore": true + }, + { + "value": "false", + "next": "editAPI", + "ignore": true + } + ] + }, + "permissionSchema": { + "key": "permissionSchema", + "type": "checkbox", + "next": "overrideSchema", + "options": ["admin", "any"], + "ignore": true + } +} diff --git a/scripts/gen-test-payload/test-helpers/vod-helpers.json b/scripts/gen-test-payload/test-helpers/vod-helpers.json deleted file mode 100644 index 0967ef42..00000000 --- a/scripts/gen-test-payload/test-helpers/vod-helpers.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/scripts/headless/add-vod.sh b/scripts/headless/add-vod.sh index 646fe659..152b8e59 100755 --- a/scripts/headless/add-vod.sh +++ b/scripts/headless/add-vod.sh @@ -12,4 +12,4 @@ VOD="{\ \"enableCMS\":false\ }" -amplify video add --payload $VOD \ No newline at end of file +amplify video add --payload $VOD diff --git a/scripts/setup.js b/scripts/setup.js index 78f2d94a..1d9f335b 100644 --- a/scripts/setup.js +++ b/scripts/setup.js @@ -17,9 +17,18 @@ async function executeScripts() { try { console.log('\namplify init'); await exec('bash', ['./scripts/headless/init-new-project.sh']); + let params = ''; + if (process.env.COVERAGE === 'full') { + params = ' full-coverage'; + } + await exec('node', [`./scripts/gen-test-payload/index.js${params}`]); console.log('\namplify add video'); - await exec('bash', ['./scripts/headless/add-ivs.sh']); - await exec('bash', ['./scripts/headless/add-vod.sh']); + const scriptsFolder = './scripts/gen-test-payload/output'; + await Promise.all(fs.readdirSync(scriptsFolder).map(async (script) => { + if (/(^\w+-\d+\.sh$)/.test(script)) { + return await exec('bash', [`${scriptsFolder}/${script}`]); + } + })); console.log('\namplify push'); await exec('bash', ['./scripts/headless/amplify-push.sh']); } catch (error) {