feat: add compile option to enable rerun of transpilation for react…

… templates (#1177)

Co-authored-by: Lukasz Gornicki <[email protected]>%0ACo-authored-by: derberg <[email protected]>

.changeset/compile_option.md

@@ -0,0 +1,5 @@
+---
+"@asyncapi/generator": minor
+---
+
+Add `compile` option to enable rerun of transpilation of templates build with react engine. It is set to `true` by default. In future major releases it will be set to `false` and we will explain how to publish template to include transpilation files by default. Transpiled files are already included in [`html-template`](https://github.com/asyncapi/html-template/pull/575). It means that you can run generator for `html-template` (it's latest version) with `compile=false` and this will improve the speed of HTML generation for you.

apps/generator/.gitignore

@@ -15,6 +15,7 @@ output
 out/
 coverage
 test/temp/integrationTestResult
+test/temp/reactTemplate
 test/test-project/package-lock.json
 test/test-project/verdaccio/storage/
 test/test-project/storage/

apps/generator/lib/generator.js

@@ -47,7 +47,7 @@ const DEFAULT_TEMPLATES_DIR = path.resolve(ROOT_DIR, 'node_modules');
 
 const TRANSPILED_TEMPLATE_LOCATION = '__transpiled';
 const TEMPLATE_CONTENT_DIRNAME = 'template';
-const GENERATOR_OPTIONS = ['debug', 'disabledHooks', 'entrypoint', 'forceWrite', 'install', 'noOverwriteGlobs', 'output', 'templateParams', 'mapBaseUrlToFolder', 'url', 'auth', 'token', 'registry'];
+const GENERATOR_OPTIONS = ['debug', 'disabledHooks', 'entrypoint', 'forceWrite', 'install', 'noOverwriteGlobs', 'output', 'templateParams', 'mapBaseUrlToFolder', 'url', 'auth', 'token', 'registry', 'compile'];
 const logMessage = require('./logMessages');
 
 const shouldIgnoreFile = filePath =>
@@ -86,20 +86,22 @@ class Generator {
    * @param {Boolean} [options.forceWrite=false] Force writing of the generated files to given directory even if it is a git repo with unstaged files or not empty dir. Default is set to false.
    * @param {Boolean} [options.install=false] Install the template and its dependencies, even when the template has already been installed.
    * @param {Boolean} [options.debug=false] Enable more specific errors in the console. At the moment it only shows specific errors about filters. Keep in mind that as a result errors about template are less descriptive.
+   * @param {Boolean} [options.compile=true] Whether to compile the template or use the cached transpiled version provided by template in '__transpiled' folder
    * @param {Object<String, String>} [options.mapBaseUrlToFolder] Optional parameter to map schema references from a base url to a local base folder e.g. url=https://schema.example.com/crm/  folder=./test/docs/ .
    * @param {Object} [options.registry]           Optional parameter with private registry configuration
    * @param {String} [options.registry.url]       Parameter to pass npm registry url
    * @param {String} [options.registry.auth]      Optional parameter to pass npm registry username and password encoded with base64, formatted like username:password value should be encoded
    * @param {String} [options.registry.token]     Optional parameter to pass npm registry auth token that you can grab from .npmrc file
    */
 
-  constructor(templateName, targetDir, { templateParams = {}, entrypoint, noOverwriteGlobs, disabledHooks, output = 'fs', forceWrite = false, install = false, debug = false, mapBaseUrlToFolder = {}, registry = {}} = {}) {
+  constructor(templateName, targetDir, { templateParams = {}, entrypoint, noOverwriteGlobs, disabledHooks, output = 'fs', forceWrite = false, install = false, debug = false, mapBaseUrlToFolder = {}, registry = {}, compile = true } = {}) {
     const options = arguments[arguments.length - 1];
     this.verifyoptions(options);
     if (!templateName) throw new Error('No template name has been specified.');
     if (!entrypoint && !targetDir) throw new Error('No target directory has been specified.');
     if (!['fs', 'string'].includes(output)) throw new Error(`Invalid output type ${output}. Valid values are 'fs' and 'string'.`);
-
+    /** @type {Boolean} Whether to compile the template or use the cached transpiled version provided by template in '__transpiled' folder. */
+    this.compile = compile;
     /** @type {Object} Npm registry information. */
     this.registry = registry;
     /** @type {String} Name of the template to generate. */
@@ -393,7 +395,7 @@ class Generator {
    * Configure the templates based the desired renderer.
    */
   async configureTemplate() {
-    if (isReactTemplate(this.templateConfig)) {
+    if (isReactTemplate(this.templateConfig) && this.compile) {
       await configureReact(this.templateDir, this.templateContentDir, TRANSPILED_TEMPLATE_LOCATION);
     } else {
       this.nunjucks = configureNunjucks(this.debug, this.templateDir);

apps/generator/lib/logMessages.js

@@ -46,6 +46,10 @@ function conditionalFilesMatched(relativeSourceFile) {
   return `${relativeSourceFile} was not generated because condition specified for this file in template configuration in conditionalFiles matched.`;
 }
 
+function compileEnabled(dir, output_dir) {
+  return `Transpilation of files ${dir} into ${output_dir} started.`;
+}
+
 module.exports = {
   TEMPLATE_INSTALL_FLAG_MSG,
   TEMPLATE_INSTALL_DISK_MSG,
@@ -59,5 +63,6 @@ module.exports = {
   templateSuccessfullyInstalled,
   relativeSourceFileNotGenerated,
   conditionalFilesMatched,
+  compileEnabled,
   skipOverwrite
 };

apps/generator/lib/renderer/react.js

@@ -16,9 +16,11 @@ const reactExport = module.exports;
  * @param {string} templateLocation located for thetemplate
  * @param {string} templateContentDir where the template content are located
  * @param {string} transpiledTemplateLocation folder for the transpiled code
+ * @param {Boolean} compile Whether to compile the template files or used the cached transpiled version provided by the template in the '__transpiled' folder
  */
 reactExport.configureReact = async (templateLocation, templateContentDir, transpiledTemplateLocation) => {
   const outputDir = path.resolve(templateLocation, `./${transpiledTemplateLocation}`);
+  log.debug(logMessage.compileEnabled(templateContentDir, outputDir));
   await AsyncReactSDK.transpileFiles(templateContentDir, outputDir, {
     recursive: true
   });

apps/generator/package.json

@@ -86,6 +86,7 @@
     "jsdoc-to-markdown": "^7.1.1",
     "markdown-toc": "^1.2.0",
     "rimraf": "^3.0.2",
-    "unixify": "^1.0.0"
+    "unixify": "^1.0.0",
+    "fs-extra": "9.1.0"
   }
 }

apps/generator/test/generator.test.js

@@ -26,6 +26,7 @@ describe('Generator', () => {
       expect(gen.forceWrite).toStrictEqual(false);
       expect(gen.install).toStrictEqual(false);
       expect(gen.templateParams).toStrictEqual({});
+      expect(gen.compile).toStrictEqual(true);
     });
 
     it('works with all the params', () => {
@@ -39,6 +40,7 @@ describe('Generator', () => {
         templateParams: {
           test: true,
         },
+        compile: false,
       });
       expect(gen.templateName).toStrictEqual('testTemplate');
       expect(gen.targetDir).toStrictEqual(__dirname);
@@ -48,6 +50,7 @@ describe('Generator', () => {
       expect(gen.output).toStrictEqual('string');
       expect(gen.forceWrite).toStrictEqual(true);
       expect(gen.install).toStrictEqual(true);
+      expect(gen.compile).toStrictEqual(false);
       expect(() => gen.templateParams.test).toThrow('Template parameter "test" has not been defined in the package.json file under generator property. Please make sure it\'s listed there before you use it in your template.');
 
       // Mock params on templateConfig so it doesn't fail.

apps/generator/test/integration.test.js

@@ -2,8 +2,9 @@
  * @jest-environment node
  */
 
-const { mkdir, writeFile, readFile } = require('fs').promises;
 const path = require('path');
+const { readFile, writeFile, access, mkdir } = require('fs').promises;
+const { copy } = require('fs-extra');
 const Generator = require('../lib/generator');
 const dummySpecPath = path.resolve(__dirname, './docs/dummy.yml');
 const refSpecPath = path.resolve(__dirname, './docs/apiwithref.json');
@@ -12,16 +13,37 @@ const crypto = require('crypto');
 const mainTestResultPath = 'test/temp/integrationTestResult';
 const reactTemplate = 'test/test-templates/react-template';
 const nunjucksTemplate = 'test/test-templates/nunjucks-template';
+//temp location where react template is copied for each test that does some mutation on template files
+const copyOfReactTemplate = 'test/temp/reactTemplate';
 
 describe('Integration testing generateFromFile() to make sure the result of the generation is not changend comparing to snapshot', () => {
   const generateFolderName = () => {
     //you always want to generate to new directory to make sure test runs in clear environment
     return path.resolve(mainTestResultPath, crypto.randomBytes(4).toString('hex'));
   };
 
-  jest.setTimeout(60000);
+  const getCleanReactTemplate = async () => {
+    //for each test new react template is needed in unique location
+    const newReactTemplateLocation = path.resolve(copyOfReactTemplate, crypto.randomBytes(4).toString('hex'));
+    await copy(reactTemplate, newReactTemplateLocation);
+    return newReactTemplateLocation;
+  };
+
+  jest.setTimeout(100000);
   const testOutputFile = 'test-file.md';
 
+  const tempJsContent = `
+  import { File, Text } from '@asyncapi/generator-react-sdk';
+  
+  export default function() {
+    return (
+      <File name="temp.md">
+        <Text>Test</Text>
+      </File>
+    );
+  }
+  `;
+
   it('generated using Nunjucks template', async () => {
     const outputDir = generateFolderName();
     const generator = new Generator(nunjucksTemplate, outputDir, {
@@ -56,8 +78,52 @@ describe('Integration testing generateFromFile() to make sure the result of the
     expect(file).toMatchSnapshot();
   });
 
+  it('check if the temp.md file is created with compile option true', async () => {
+    const outputDir = generateFolderName();
+    const cleanReactTemplate = await getCleanReactTemplate();
+    // Create temp.md.js file dynamically
+
+    const tempJsPath = path.join(cleanReactTemplate, 'template/temp.md.js');
+    // Create temp.md.js file dynamically
+    await writeFile(tempJsPath, tempJsContent);
+
+    const generator = new Generator(cleanReactTemplate, outputDir, {
+      forceWrite: true,
+      compile: true,
+      debug: true,
+    });
+    await generator.generateFromFile(dummySpecPath);
+
+    const tempMdPath = path.join(outputDir, 'temp.md');
+
+    // Check the content of temp.md
+    const tempMdContent = await readFile(tempMdPath, 'utf8');
+    expect(tempMdContent.trim()).toBe('Test');
+  });
+
+  it('check if the temp.md file is not created when compile option is false', async () => {
+    const outputDir = generateFolderName();
+    const cleanReactTemplate = await getCleanReactTemplate();
+    // Create temp.md.js file dynamically
+    const tempJsPath = path.join(cleanReactTemplate, 'template/temp.md.js');
+    await writeFile(tempJsPath, tempJsContent);
+
+    const generator = new Generator(cleanReactTemplate, outputDir, {
+      forceWrite: true,
+      compile: false, 
+      debug: true
+    });
+    await generator.generateFromFile(dummySpecPath);
+
+    // Check if temp.md is not created in the output directory
+    const tempMdPath = path.join(outputDir, 'temp.md');
+    const tempMdExists = await access(tempMdPath).then(() => true).catch(() => false);
+    expect(tempMdExists).toBe(false);
+  });
+
   it('should ignore specified files with noOverwriteGlobs', async () => {
     const outputDir = generateFolderName();
+    const cleanReactTemplate = await getCleanReactTemplate();
     // Manually create a file to test if it's not overwritten
     await mkdir(outputDir, { recursive: true });
     // Create a variable to store the file content
@@ -67,7 +133,7 @@ describe('Integration testing generateFromFile() to make sure the result of the
     await writeFile(testFilePath, testContent);
 
     // Manually create an output first, before generation, with additional custom file to validate if later it is still there, not overwritten
-    const generator = new Generator(reactTemplate, outputDir, {
+    const generator = new Generator(cleanReactTemplate, outputDir, {
       forceWrite: true,
       noOverwriteGlobs: [`**/${testOutputFile}`],
       debug: true,

apps/generator/test/test-project/test-project.test.js

@@ -41,7 +41,7 @@ describe('Testing if markdown was generated with proper version of the template'
   it('Test B - generated markdown should contain new content because of explicit fresh installation of different template version (install: true)', async () => {
     const templateVersion = '0.0.2';
 
-    const generator = new Generator(`${templateName}@${templateVersion}`, tempOutputResults, { forceWrite: true, install: true, debug: true, templateParams: { version: 'v1', mode: 'production' } });
+    const generator = new Generator(`${templateName}@${templateVersion}`, tempOutputResults, { compile: true, forceWrite: true, install: true, debug: true, templateParams: { version: 'v1', mode: 'production' } });
     await generator.generateFromFile(dummySpecPath);
 
     const file = await readFile(path.join(tempOutputResults, fileToCheck), 'utf8');
@@ -95,4 +95,4 @@ describe('Testing if markdown was generated with proper version of the template'
     expect(console.log).toHaveBeenCalledWith(logMessage.templateVersion(version));
     expect(console.log).toHaveBeenCalledWith(logMessage.NPM_INSTALL_TRIGGER);
   });
-});
+});

apps/generator/test/test-project/test-registry.test.js

@@ -16,6 +16,7 @@ describe('Integration testing generateFromFile() to make sure the template can b
   it('generated using private registory', async () => {
     const generator = new Generator('react-template', tempOutputResults,
       {
+        compile: true,
         debug: true,
         install: true,
         forceWrite: true,
@@ -41,6 +42,7 @@ describe('Integration testing generateFromFile() to make sure the template can b
   it('generated using private registory from npm config', async () => {
     const generator = new Generator('react-template', tempOutputResults,
       {
+        compile: true,
         debug: true,
         install: true,
         forceWrite: true,