feat: 2647.2 add secure template repos to pfe

docs/openapi.yml

@@ -1139,6 +1139,10 @@ paths:
                   $ref: '#/components/schemas/TemplateRepo'
         400:
           description: Bad request
+          content:
+            text/plain:
+              schema:
+                type: string
         423:
           description: The templates object is currently locked to prevent incorrect data being accessed and returned. Retry later
     delete:
@@ -1989,6 +1993,13 @@ components:
           type: boolean
         protected:
           type: boolean
+        gitCredentials:
+          type: object
+          properties:
+            username:
+              type: string
+            password:
+              type: string
     TemplateRepo:
       type: object
       required:

src/pfe/portal/modules/ExtensionList.js

@@ -190,7 +190,10 @@ async function addExtensionsToTemplates(extensions, templates) {
     try {
       if (extension.templates) {
         log.trace(`Adding Extension ${extension.name}'s repository into the templates`);
-        await templates.addRepository(extension.templates, extension.description);
+        await templates.addRepository({
+          url: extension.templates,
+          description: extension.description,
+        });
       } else if (extension.templatesProvider) {
         log.trace(`Adding Extension ${extension.name}'s provider into the templates`);
         await templates.addProvider(extension.name, extension.templatesProvider);

src/pfe/portal/modules/Templates.js

@@ -147,12 +147,19 @@ module.exports = class Templates {
   /**
    * Add a repository to the list of template repositories.
    */
-  async addRepository(repoUrl, repoDescription, repoName, isRepoProtected = false, isRepoEnabled = true) {
+  async addRepository({
+    url,
+    name,
+    description,
+    gitCredentials,
+    protected: _protected = false, // the keyword 'protected' is reserved
+    enabled = true,
+  }) {
     this.lock();
     try {
       const repositories = cwUtils.deepClone(this.repositoryList);
-      const validatedUrl = await validateRepository(repoUrl, repositories);
-      const newRepo = await constructRepositoryObject(validatedUrl, repoDescription, repoName, isRepoProtected, isRepoEnabled);
+      const validatedUrl = await validateRepository(url, repositories, gitCredentials);
+      const newRepo = await constructRepositoryObject(validatedUrl, description, name, _protected, enabled, gitCredentials);
 
       await addRepositoryToProviders(newRepo, this.providers);
 
@@ -162,9 +169,9 @@ module.exports = class Templates {
       this.repositoryList = await updateRepoListWithReposFromProviders(this.providers, newRepositoryList, this.repositoryFile);
 
       // Fetch the repository templates and add them appropriately
-      const newTemplates = await getTemplatesFromRepo(newRepo)
+      const newTemplates = await getTemplatesFromRepo(newRepo, gitCredentials)
       this.allProjectTemplates = this.allProjectTemplates.concat(newTemplates);
-      if (isRepoEnabled) {
+      if (enabled) {
         this.enabledProjectTemplates = this.enabledProjectTemplates.concat(newTemplates);
       }
       await writeRepositoryList(this.repositoryFile, this.repositoryList);
@@ -256,7 +263,7 @@ async function writeRepositoryList(repositoryFile, repositoryList) {
   log.info(`Repository list updated.`);
 }
 
-async function validateRepository(repoUrl, repositories) {
+async function validateRepository(repoUrl, repositories, gitCredentials) {
   let url;
   try {
     url = new URL(repoUrl).href;
@@ -271,23 +278,23 @@ async function validateRepository(repoUrl, repositories) {
     throw new TemplateError('DUPLICATE_URL', repoUrl);
   }
 
-  const validJsonURL = await doesURLPointToIndexJSON(url);
-  if (!validJsonURL) {
-    throw new TemplateError('URL_DOES_NOT_POINT_TO_INDEX_JSON', url);
+  const templateSummaries = await getTemplateSummaries(repoUrl, gitCredentials);
+  if (!templateSummaries.length || templateSummaries.some(summary => !isTemplateSummary(summary))) {
+    throw new TemplateError('URL_DOES_NOT_POINT_TO_INDEX_JSON', repoUrl);
   }
 
   return url;
 }
 
-async function constructRepositoryObject(url, description, name, isRepoProtected, isRepoEnabled) {
+async function constructRepositoryObject(url, description, name, isRepoProtected, isRepoEnabled, gitCredentials) {
   let repository = {
     id: uuidv5(url, uuidv5.URL),
     name,
     url,
     description,
     enabled: isRepoEnabled,
   }
-  repository = await fetchRepositoryDetails(repository);
+  repository = await fetchRepositoryDetails(repository, gitCredentials);
   if (isRepoProtected !== undefined) {
     repository.protected = isRepoProtected;
   }
@@ -325,65 +332,67 @@ function fetchAllRepositoryDetails(repos) {
   );
 }
 
-async function fetchRepositoryDetails(repo) {
+async function fetchRepositoryDetails(repo, gitCredentials) {
   let newRepo = cwUtils.deepClone(repo);
 
   // Only set the name or description of the repo if not given by the user
   if (!(repo.name && repo.description)){
-    const repoDetails = await getNameAndDescriptionFromRepoTemplatesJSON(newRepo.url);
+    const repoDetails = await getNameAndDescriptionFromRepoTemplatesJSON(newRepo.url, gitCredentials);
     newRepo = cwUtils.updateObject(newRepo, repoDetails);
   }
 
   if (repo.projectStyles) {
     return newRepo;
   }
 
-  const templatesFromRepo = await getTemplatesFromRepo(repo);
+  const templatesFromRepo = await getTemplatesFromRepo(repo, gitCredentials);
   newRepo.projectStyles = getTemplateStyles(templatesFromRepo);
   return newRepo;
 }
 
-async function getNameAndDescriptionFromRepoTemplatesJSON(url) {
+async function getNameAndDescriptionFromRepoTemplatesJSON(url, gitCredentials) {
   if (!url) throw new Error(`must supply a URL`);
 
   const templatesUrl = new URL(url);
   // return repository untouched if repository url points to a local file
-  if ( templatesUrl.protocol === 'file:' ) {
+  if (templatesUrl.protocol === 'file:') {
     return {};
   }
-  const indexPath = templatesUrl.pathname;
-  const templatesPath = path.dirname(indexPath) + '/' + 'templates.json';
-
-  templatesUrl.pathname = templatesPath;
-
-  const options = {
-    host: templatesUrl.hostname,
-    path: templatesUrl.pathname,
-    port: templatesUrl.port,
-    method: 'GET',
-  }
+  templatesUrl.pathname = `${path.dirname(templatesUrl.pathname)}/templates.json`;
 
-  const res = await cwUtils.asyncHttpRequest(options, undefined, templatesUrl.protocol === 'https:');
+  const res = await makeGetRequest(templatesUrl, gitCredentials);
   if (res.statusCode !== 200) {
     return {};
   }
 
   try {
-    const templateDetails = JSON.parse(res.body);
-    const repositoryDetails = {};
-    for (const prop of ['name', 'description']) {
-      if (templateDetails.hasOwnProperty(prop)) {
-        repositoryDetails[prop] = templateDetails[prop];
-      }
-    }
-    return repositoryDetails;
+    const { name, description } = JSON.parse(res.body);
+    return { name, description };
   } catch (error) {
     // Log an error but don't throw an exception as this is optional.
     log.error(`URL '${templatesUrl}' should return JSON`);
   }
   return {};
 }
 
+async function makeGetRequest(url, gitCredentials) {
+  const options = {
+    host: url.hostname,
+    path: url.pathname,
+    port: url.port,
+    method: 'GET',
+  }
+  if (gitCredentials && gitCredentials.username && gitCredentials.password) {
+    options.auth = `${gitCredentials.username}:${gitCredentials.password}`;
+  }
+  const res = await cwUtils.asyncHttpRequest(
+    options,
+    undefined,
+    url.protocol === 'https:',
+  );
+  return res;
+}
+
 /**
  * Uses a template's sourceId, sourceURL or source (repo name) to get the repository for a given template
  * Ordered so that the source (repo name) is used as a fall back
@@ -436,12 +445,12 @@ async function getTemplatesFromRepos(repos) {
   return newProjectTemplates;
 }
 
-async function getTemplatesFromRepo(repository) {
+async function getTemplatesFromRepo(repository, gitCredentials) {
   if (!repository.url) {
     throw new Error(`repo '${repository}' must have a URL`);
   }
 
-  const templateSummaries = await getTemplatesJSONFromURL(repository.url);
+  const templateSummaries = await getTemplateSummaries(repository.url, gitCredentials);
 
   const templates = templateSummaries.map(summary => {
     const template = {
@@ -470,7 +479,7 @@ async function getTemplatesFromRepo(repository) {
   return templates;
 }
 
-async function getTemplatesJSONFromURL(givenURL) {
+async function getTemplateSummaries(givenURL, gitCredentials) {
   const parsedURL = new URL(givenURL);
   let templateSummaries;
   // check if repository url points to a local file and read it accordingly
@@ -480,23 +489,17 @@ async function getTemplatesJSONFromURL(givenURL) {
         templateSummaries = await fs.readJSON(parsedURL.pathname);
       }
     } catch (err) {
-      throw new Error(`repo file '${parsedURL}' did not return JSON`);
+      throw new TemplateError('URL_DOES_NOT_POINT_TO_INDEX_JSON', null, `repo file '${parsedURL}' did not return JSON`);
     }
   } else {
-    const options = {
-      host: parsedURL.hostname,
-      path: parsedURL.pathname,
-      port: parsedURL.port,
-      method: 'GET',
-    }
-    const res = await cwUtils.asyncHttpRequest(options, undefined, parsedURL.protocol === 'https:');
+    const res = await makeGetRequest(parsedURL, gitCredentials);
     if (res.statusCode !== 200) {
-      throw new Error(`Unexpected HTTP status for ${givenURL}: ${res.statusCode}`);
+      throw new TemplateError('URL_DOES_NOT_POINT_TO_INDEX_JSON', null, `Unexpected HTTP status for ${givenURL}: ${res.statusCode}`);
     }
     try {
       templateSummaries = JSON.parse(res.body);
     } catch (error) {
-      throw new Error(`URL '${parsedURL}' did not return JSON`);
+      throw new TemplateError('URL_DOES_NOT_POINT_TO_INDEX_JSON', null, `URL '${parsedURL}' did not return JSON`);
     }
   }
   return templateSummaries;
@@ -545,19 +548,6 @@ function isRepo(obj) {
   return Boolean((obj && obj.hasOwnProperty('url')));
 }
 
-async function doesURLPointToIndexJSON(inputUrl) {
-  try {
-    const templateSummaries = await getTemplatesJSONFromURL(inputUrl);
-    if (templateSummaries.some(summary => !isTemplateSummary(summary))) {
-      return false;
-    }
-  } catch(error) {
-    log.warn(error);
-    return false
-  }
-  return true;
-}
-
 function isTemplateSummary(obj) {
   const expectedKeys = ['displayName', 'description', 'language', 'projectType', 'location', 'links'];
   return expectedKeys.every(key => obj.hasOwnProperty(key));

src/pfe/portal/routes/templates.route.js

@@ -62,24 +62,22 @@ router.get('/api/v1/templates/repositories', async (req, res, _next) => {
   }
 });
 
-router.post('/api/v1/templates/repositories', validateReq, async (req, res, _next) => {
+router.post('/api/v1/templates/repositories', validateReq, postRepositories);
+
+async function postRepositories(req, res, next) {
   const { templates: templatesController } = req.cw_user;
-  const repositoryName = req.sanitizeBody('name')
-  const repositoryUrl = req.sanitizeBody('url');
-  const repositoryDescription = req.sanitizeBody('description');
-  const isRepoProtected = req.sanitizeBody('protected');
-  // Enabled should default to true if it isn't passed.
-  const isRepoEnabled = req.sanitizeBody('enabled') !== false;
+  const repoOptions = {
+    name: req.sanitizeBody('name'),
+    url: req.sanitizeBody('url'),
+    description: req.sanitizeBody('description'),
+    protected: req.sanitizeBody('protected'),
+    enabled: req.sanitizeBody('enabled') !== false, // Enabled should default to true if it isn't passed
+    gitCredentials: req.sanitizeBody('gitCredentials'),
+  };
 
   try {
-    await templatesController.addRepository(
-      repositoryUrl,
-      repositoryDescription,
-      repositoryName,
-      isRepoProtected,
-      isRepoEnabled
-    );
-    await sendRepositories(req, res, _next);
+    await templatesController.addRepository(repoOptions);
+    await sendRepositories(req, res, next);
   } catch (error) {
     log.error(error);
     const knownErrorCodes = ['INVALID_URL', 'DUPLICATE_URL', 'URL_DOES_NOT_POINT_TO_INDEX_JSON', 'ADD_TO_PROVIDER_FAILURE'];
@@ -94,7 +92,7 @@ router.post('/api/v1/templates/repositories', validateReq, async (req, res, _nex
     }
     res.status(500).send(error.message);
   }
-});
+}
 
 router.delete('/api/v1/templates/repositories', validateReq, async (req, res, _next) => {
   const { templates: templatesController } = req.cw_user;

test/modules/template.service.js

@@ -33,6 +33,12 @@ const styledTemplates = {
     },
 };
 
+// Insert your own credentials to run relevant tests
+const gheCredentials = {
+    username: '[email protected]',
+    // password: 'INSERT_TO_RUN_RELEVANT_TESTS',
+};
+
 const sampleRepos = {
     codewind: {
         url: templateRepositoryURL,
@@ -50,6 +56,11 @@ const sampleRepos = {
         projectStyles: ['Codewind'],
         name: 'Default disabled templates',
     },
+    GHE: {
+        url: 'https://raw.github.ibm.com/Richard-Waller/sampleGHETemplateRepo/415ece47958250175f182c095af7da6cfe40e58a/devfiles/index.json',
+        description: 'Example GHE template repository',
+        name: 'Example GHE template repository',
+    },
 };
 
 const validUrlNotPointingToIndexJson = 'https://support.oneskyapp.com/hc/en-us/article_attachments/202761627/example_1.json';
@@ -80,11 +91,11 @@ async function getTemplateRepos() {
     return res;
 }
 
-async function addTemplateRepo(repo) {
+async function addTemplateRepo(repoOptions) {
     const res = await reqService.chai
         .post('/api/v1/templates/repositories')
         .set('Cookie', ADMIN_COOKIE)
-        .send(repo);
+        .send(repoOptions);
     return res;
 }
 
@@ -169,18 +180,22 @@ async function getNumberOfEnabledTemplates(queryParams) {
 }
 
 
+const idOfImmutableRepo = 'incubator';
 /**
  * Removes all templates repos known to PFE, and adds the supplied repos
  * @param {[JSON]} repoList
  */
 async function setTemplateReposTo(repoList) {
-    const reposToDelete = (await getTemplateRepos()).body;
-    if (reposToDelete.length > 0) {
+    const currentRepos = (await getTemplateRepos()).body;
+    if (currentRepos.length > 0) {
+        const reposToDelete = currentRepos.filter(repo => repo.id !== idOfImmutableRepo);
+        console.log(reposToDelete);
         for (const repo of reposToDelete) {
             await deleteTemplateRepo(repo.url);
         }
     }
-    for (const repo of repoList) {
+    const templatesToAdd = repoList.filter(repo => repo.id !== idOfImmutableRepo);
+    for (const repo of templatesToAdd) {
         await addTemplateRepo(repo);
     }
 }
@@ -241,6 +256,7 @@ module.exports = {
     templateRepositoryURL,
     sampleRepos,
     validUrlNotPointingToIndexJson,
+    gheCredentials,
     getTemplateRepos,
     addTemplateRepo,
     deleteTemplateRepo,