Support project update via GH app JWT (#10332)

Implements functionality for IR-2465

If project.update is called with a GitHub App JWT in the body (passed
as 'token'), this will trigger alternative logic in the authenticate hook.
It will check that the token comes from the same GH App that's registered
with the deployment and that the token is valid. This will bypass user
authentication, and put the token on context.params.appJWT.

updateProject will use this token to generate an installation access token
for the repo the project is in (throwing an error if the app is not
installed such that it can access that repo), then use that token to do
the git clone instead of a user OAuth token.

Updated GitHub admin authentication-settings to allow for entry of AppID.
Also updated OAuth authentication-setting schema to account for appId.

Fixed some minor bugs with project name usage when calling project.update
from the front-end.

packages/client-core/i18n/en/admin.json

@@ -355,6 +355,7 @@
         "subtitle": "Edit Authentication Settings"
       },
       "service": "Service",
+      "githubAppId": "App ID (Enter for GitHub App, omit for OAuth App)",
       "secret": "Secret",
       "entity": "Entity",
       "authStrategies": "Authentication Strategies",

packages/client-core/src/admin/components/project/ProjectTable.tsx

@@ -90,7 +90,7 @@ export default function ProjectTable() {
       await ProjectService.uploadProject({
         sourceURL: projectUpdateStatus.sourceURL,
         destinationURL: projectUpdateStatus.destinationURL,
-        name: projectUpdateStatus.projectName,
+        name: project.name,
         reset: true,
         commitSHA: projectUpdateStatus.selectedSHA,
         sourceBranch: projectUpdateStatus.selectedBranch,

packages/client-core/src/admin/components/settings/tabs/authentication.tsx

@@ -143,6 +143,16 @@ const AuthenticationTab = forwardRef(({ open }: { open: boolean }, ref: React.Mu
     state.set(temp)
   }
 
+  const handleOnChangeAppId = (event, type) => {
+    keySecret.set({
+      ...JSON.parse(JSON.stringify(keySecret.value)),
+      [type]: {
+        ...JSON.parse(JSON.stringify(keySecret[type].value)),
+        appId: event.target.value
+      }
+    })
+  }
+
   const handleOnChangeKey = (event, type) => {
     keySecret.set({
       ...JSON.parse(JSON.stringify(keySecret.value)),
@@ -371,6 +381,12 @@ const AuthenticationTab = forwardRef(({ open }: { open: boolean }, ref: React.Mu
               {t('admin:components.setting.github')}
             </Text>
 
+            <PasswordInput
+              label={t('admin:components.setting.githubAppId')}
+              value={keySecret?.value?.github?.appId || ''}
+              onChange={(e) => handleOnChangeAppId(e, OAUTH_TYPES.GITHUB)}
+            />
+
             <PasswordInput
               label={t('admin:components.setting.key')}
               value={keySecret?.value?.github?.key || ''}

packages/client-core/src/admin/services/ProjectUpdateService.ts

@@ -72,7 +72,7 @@ export const ProjectUpdateService = {
       sourceProjectName: '',
       sourceVsDestinationChecked: false,
       selectedSHA: '',
-      projectName: '',
+      projectName: projectName,
       submitDisabled: true,
       triggerSetDestination: '',
       updateType: 'none' as ProjectType['updateType'],

packages/common/src/constants/GitHubConstants.ts

@@ -26,3 +26,4 @@ Ethereal Engine. All Rights Reserved.
 export const GITHUB_URL_REGEX = /(?:git@|https:\/\/)([a-zA-Z0-9\-]+:[a-zA-Z0-9_]+@)?github.com[:/](.*)[.git]?/
 export const GITHUB_PER_PAGE = 100
 export const PUBLIC_SIGNED_REGEX = /https:\/\/[\w\d\s\-_]+:[\w\d\s\-_]+@github.com\/([\w\d\s\-_]+)\/([\w\d\s\-_]+).git/
+export const INSTALLATION_SIGNED_REGEX = /https:\/\/oauth2:[\w\d\s\-_]+@github.com\/([\w\d\s\-_]+)\/([\w\d\s\-_]+).git/

packages/common/src/schemas/projects/project-build.schema.ts

@@ -39,7 +39,8 @@ export const ProjectBuildUpdateItemSchema = Type.Object(
     commitSHA: Type.String(),
     sourceBranch: Type.String(),
     updateType: StringEnum(projectUpdateTypes),
-    updateSchedule: Type.String()
+    updateSchedule: Type.String(),
+    token: Type.Optional(Type.Boolean())
   },
   { $id: 'ProjectUpdate', additionalProperties: false }
 )

packages/common/src/schemas/setting/authentication-setting.schema.ts

@@ -89,6 +89,7 @@ export interface AuthDefaultsType extends Static<typeof authDefaultsSchema> {}
 
 export const authAppCredentialsSchema = Type.Object(
   {
+    appId: Type.Optional(Type.String()),
     key: Type.String(),
     secret: Type.String(),
     scope: Type.Optional(Type.Array(Type.String())),

packages/server-core/src/appconfig.ts

@@ -306,6 +306,7 @@ const authentication = {
       secret: process.env.FACEBOOK_CLIENT_SECRET!
     },
     github: {
+      appId: process.env.GITHUB_APP_ID!,
       key: process.env.GITHUB_CLIENT_ID!,
       secret: process.env.GITHUB_CLIENT_SECRET!,
       scope: GITHUB_SCOPES

packages/server-core/src/hooks/authenticate.ts

@@ -24,16 +24,19 @@ Ethereal Engine. All Rights Reserved.
 */
 
 import * as authentication from '@feathersjs/authentication'
+import { NotAuthenticated } from '@feathersjs/errors'
 import { HookContext, NextFunction, Paginated } from '@feathersjs/feathers'
+import { Octokit } from '@octokit/rest'
 import { AsyncLocalStorage } from 'async_hooks'
 import { isProvider } from 'feathers-hooks-common'
 
 import { userApiKeyPath, UserApiKeyType } from '@etherealengine/common/src/schemas/user/user-api-key.schema'
 import { userPath, UserType } from '@etherealengine/common/src/schemas/user/user.schema'
 import { toDateTimeSql } from '@etherealengine/common/src/utils/datetime-sql'
 
+import { decode, JwtPayload } from 'jsonwebtoken'
+import { Application } from '../../declarations'
 import config from '../appconfig'
-import { Application } from './../../declarations'
 
 const { authenticate } = authentication.hooks
 
@@ -65,6 +68,26 @@ export default async (context: HookContext<Application>, next: NextFunction): Pr
     return next()
   }
 
+  if (context.arguments[1]?.token && context.path === 'project' && context.method === 'update') {
+    const appId = config.authentication.oauth.github.appId ? parseInt(config.authentication.oauth.github.appId) : null
+    const token = context.arguments[1].token
+    const jwtDecoded = decode(token)! as JwtPayload
+    if (jwtDecoded.iss == null || parseInt(jwtDecoded.iss) !== appId)
+      throw new NotAuthenticated('Invalid app credentials')
+    const octokit = new Octokit({ auth: token })
+    let appResponse
+    try {
+      appResponse = await octokit.rest.apps.getAuthenticated()
+    } catch (err) {
+      throw new NotAuthenticated('Invalid app credentials')
+    }
+    if (appResponse.data.id !== appId) throw new NotAuthenticated('App ID of JWT does not match installed App ID')
+    context.params.appJWT = token
+    context.params.signedByAppJWT = true
+    delete context.arguments[1].token
+    return next()
+  }
+
   // Ignore whitelisted services & methods
   const isWhitelisted = checkWhitelist(context)
   if (isWhitelisted) {

packages/server-core/src/hooks/is-signed-by-app-jwt.ts

@@ -0,0 +1,35 @@
+/*
+CPAL-1.0 License
+
+The contents of this file are subject to the Common Public Attribution License
+Version 1.0. (the "License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+https://github.com/EtherealEngine/etherealengine/blob/dev/LICENSE.
+The License is based on the Mozilla Public License Version 1.1, but Sections 14
+and 15 have been added to cover use of software over a computer network and 
+provide for limited attribution for the Original Developer. In addition, 
+Exhibit A has been modified to be consistent with Exhibit B.
+
+Software distributed under the License is distributed on an "AS IS" basis,
+WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the
+specific language governing rights and limitations under the License.
+
+The Original Code is Ethereal Engine.
+
+The Original Developer is the Initial Developer. The Initial Developer of the
+Original Code is the Ethereal Engine team.
+
+All portions of the code written by the Ethereal Engine team are Copyright © 2021-2023 
+Ethereal Engine. All Rights Reserved.
+*/
+
+import { HookContext } from '../../declarations'
+
+/**
+ * Hook used to check if request is signed by App JWT
+ */
+export const isSignedByAppJWT = () => {
+  return (context: HookContext) => {
+    return context.params.signedByAppJWT
+  }
+}

packages/server-core/src/projects/project/github-helper.ts

@@ -67,9 +67,12 @@ const TOKEN_REGEX = /"RemoteAuth ([0-9a-zA-Z-_]+)"/
 const OID_REGEX = /oid sha256:([0-9a-fA-F]{64})/
 const PUSH_TIMEOUT = 60 * 10 //10 minute timeout on GitHub push jobs completing or failing
 
-export const getAuthenticatedRepo = async (token: string, repositoryPath: string) => {
+export const getAuthenticatedRepo = async (token: string, repositoryPath: string, isInstallationToken = false) => {
   try {
     if (!/.git$/.test(repositoryPath)) repositoryPath = repositoryPath + '.git'
+    if (isInstallationToken) {
+      return repositoryPath.replace('https://', `https://oauth2:${token}@`)
+    }
     const user = await getUser(token)
     return repositoryPath.replace('https://', `https://${user.data.login}:${token}@`)
   } catch (error) {

packages/server-core/src/projects/project/project-helper.ts

@@ -30,10 +30,10 @@ import {
 } from '@aws-sdk/client-ecr'
 import { DescribeImagesCommand, ECRPUBLICClient } from '@aws-sdk/client-ecr-public'
 import { fromIni } from '@aws-sdk/credential-providers'
-import { BadRequest, Forbidden } from '@feathersjs/errors'
+import { BadRequest, Forbidden, NotFound } from '@feathersjs/errors'
 import { Paginated } from '@feathersjs/feathers'
 import * as k8s from '@kubernetes/client-node'
-import { RestEndpointMethodTypes } from '@octokit/rest'
+import { Octokit, RestEndpointMethodTypes } from '@octokit/rest'
 import appRootPath from 'app-root-path'
 import { exec } from 'child_process'
 import { compareVersions } from 'compare-versions'
@@ -45,7 +45,7 @@ import { promisify } from 'util'
 import { v4 as uuidv4 } from 'uuid'
 
 import { AssetType } from '@etherealengine/common/src/constants/AssetType'
-import { PUBLIC_SIGNED_REGEX } from '@etherealengine/common/src/constants/GitHubConstants'
+import { INSTALLATION_SIGNED_REGEX, PUBLIC_SIGNED_REGEX } from '@etherealengine/common/src/constants/GitHubConstants'
 import { ManifestJson } from '@etherealengine/common/src/interfaces/ManifestJson'
 import { ProjectPackageJsonType } from '@etherealengine/common/src/interfaces/ProjectPackageJsonType'
 import { apiJobPath } from '@etherealengine/common/src/schemas/cluster/api-job.schema'
@@ -989,17 +989,16 @@ export async function getProjectUpdateJobBody(
     updateSchedule: string
   },
   app: Application,
-  userId: string,
-  jobId: string
+  jobId: string,
+  userId?: string,
+  token?: string
 ): Promise<k8s.V1Job> {
   const command = [
     'npx',
     'cross-env',
     'ts-node',
     '--swc',
     'scripts/update-project.ts',
-    `--userId`,
-    userId,
     '--sourceURL',
     data.sourceURL,
     '--destinationURL',
@@ -1008,13 +1007,25 @@ export async function getProjectUpdateJobBody(
     data.name,
     '--sourceBranch',
     data.sourceBranch,
-    '--updateType',
-    data.updateType,
-    '--updateSchedule',
-    data.updateSchedule,
     '--jobId',
     jobId
   ]
+  if (data.updateType) {
+    command.push('--updateType')
+    command.push(data.updateType as string)
+  }
+  if (data.updateSchedule) {
+    command.push('--updateSchedule')
+    command.push(data.updateSchedule)
+  }
+  if (token) {
+    command.push('--token')
+    command.push(token)
+  }
+  if (userId) {
+    command.push('--userId')
+    command.push(userId)
+  }
   if (data.commitSHA) {
     command.push('--commitSHA')
     command.push(data.commitSHA)
@@ -1327,6 +1338,7 @@ export const updateProject = async (
     sourceBranch: string
     updateType: ProjectType['updateType']
     updateSchedule: string
+    token?: string
   },
   params?: ProjectParams
 ) => {
@@ -1374,24 +1386,50 @@ export const updateProject = async (
     }
   })) as Paginated<ProjectType>
 
-  let project
+  let project, userId
   if (projectResult.data.length > 0) project = projectResult.data[0]
 
-  const userId = params!.user?.id || project?.updateUserId
-  if (!userId) throw new BadRequest('No user ID from call or existing project owner')
-
-  const githubIdentityProvider = (await app.service(identityProviderPath).find({
-    query: {
-      userId: userId,
-      type: 'github',
-      $limit: 1
+  let repoPath,
+    signingToken,
+    usesInstallationToken = false
+  if (params?.appJWT) {
+    const octokit = new Octokit({ auth: params.appJWT })
+    let repoInstallation
+    try {
+      repoInstallation = await octokit.rest.apps.getRepoInstallation({
+        owner: urlParts[urlParts.length - 2],
+        repo: urlParts[urlParts.length - 1]
+      })
+    } catch (err) {
+      throw new NotFound(
+        'The GitHub App associated with this deployment has not been installed with access to that repository, or that repository does not exist'
+      )
     }
-  })) as Paginated<IdentityProviderType>
+    const installationAccessToken = await octokit.rest.apps.createInstallationAccessToken({
+      installation_id: repoInstallation.data.id
+    })
+    signingToken = installationAccessToken.data.token
+    usesInstallationToken = true
+    repoPath = await getAuthenticatedRepo(signingToken, data.sourceURL, usesInstallationToken)
+    params.provider = 'server'
+  } else {
+    userId = params!.user?.id || project?.updateUserId
+    if (!userId) throw new BadRequest('No user ID from call or existing project owner')
+
+    const githubIdentityProvider = (await app.service(identityProviderPath).find({
+      query: {
+        userId: userId,
+        type: 'github',
+        $limit: 1
+      }
+    })) as Paginated<IdentityProviderType>
 
-  if (githubIdentityProvider.data.length === 0) throw new Forbidden('You are not authorized to access this project')
+    if (githubIdentityProvider.data.length === 0) throw new Forbidden('You are not authorized to access this project')
 
-  let repoPath = await getAuthenticatedRepo(githubIdentityProvider.data[0].oauthToken!, data.sourceURL)
-  if (!repoPath) repoPath = data.sourceURL //public repo
+    signingToken = githubIdentityProvider.data[0].oauthToken
+    repoPath = await getAuthenticatedRepo(signingToken, data.sourceURL)
+    if (!repoPath) repoPath = data.sourceURL //public repo
+  }
 
   const gitCloner = useGit(projectLocalDirectory)
   await gitCloner.clone(repoPath, projectDirectory)
@@ -1439,9 +1477,13 @@ export const updateProject = async (
   const existingProject = existingProjectResult.total > 0 ? existingProjectResult.data[0] : null
   let repositoryPath = data.destinationURL || data.sourceURL
   const publicSignedExec = PUBLIC_SIGNED_REGEX.exec(repositoryPath)
+  const installationSignedExec = INSTALLATION_SIGNED_REGEX.exec(repositoryPath)
   //In testing, intermittently the signed URL was being entered into the database, which made matching impossible.
   //Stripping the signed portion out if it's about to be inserted.
+  if (installationSignedExec)
+    repositoryPath = `https://github.com/${installationSignedExec[1]}/${installationSignedExec[2]}`
   if (publicSignedExec) repositoryPath = `https://github.com/${publicSignedExec[1]}/${publicSignedExec[2]}`
+
   const { commitSHA, commitDate } = await getCommitSHADate(projectName)
 
   const returned = !existingProject
@@ -1458,7 +1500,7 @@ export const updateProject = async (
           sourceBranch: data.sourceBranch,
           updateType: data.updateType,
           updateSchedule: data.updateSchedule,
-          updateUserId: userId,
+          updateUserId: userId || null,
           commitSHA,
           commitDate: toDateTimeSql(commitDate),
           assetsOnly: assetsOnly,
@@ -1479,7 +1521,7 @@ export const updateProject = async (
           sourceBranch: data.sourceBranch,
           updateType: data.updateType,
           updateSchedule: data.updateSchedule,
-          updateUserId: userId
+          updateUserId: userId || null
         },
         params
       )
@@ -1492,7 +1534,7 @@ export const updateProject = async (
     })
 
   if (data.reset) {
-    let repoPath = await getAuthenticatedRepo(githubIdentityProvider.data[0].oauthToken!, data.destinationURL)
+    let repoPath = await getAuthenticatedRepo(signingToken, data.destinationURL, usesInstallationToken)
     if (!repoPath) repoPath = data.destinationURL //public repo
     await git.addRemote('destination', repoPath)
     await git.raw(['lfs', 'fetch', '--all'])

packages/server-core/src/projects/project/project.class.ts

@@ -64,7 +64,9 @@ const UPDATE_JOB_TIMEOUT = 60 * 5 //5 minute timeout on project update jobs comp
 
 const projectsRootFolder = path.join(appRootPath.path, 'packages/projects/projects/')
 
-export interface ProjectParams extends KnexAdapterParams<ProjectQuery>, ProjectUpdateParams {}
+export interface ProjectParams extends KnexAdapterParams<ProjectQuery>, ProjectUpdateParams {
+  appJWT?: string
+}
 
 export type ProjectParamsClient = Omit<ProjectParams, 'user'>