From 1b98060acea98f057649dd67ee4bffc1df95d303 Mon Sep 17 00:00:00 2001 From: ghe Date: Mon, 14 Nov 2022 12:08:15 +0000 Subject: [PATCH] feat: sync support for github enterprise --- docs/sync.md | 22 ++ src/cmds/sync.ts | 1 - src/lib/types.ts | 7 +- src/scripts/sync/sync-org-projects.ts | 9 +- src/scripts/sync/sync-projects-per-target.ts | 5 +- test/scripts/sync/sync-org-projects.test.ts | 335 ++++++++++++++++++- test/system/__snapshots__/sync.test.ts.snap | 2 +- test/system/sync.test.ts | 57 +--- 8 files changed, 359 insertions(+), 79 deletions(-) diff --git a/docs/sync.md b/docs/sync.md index 295788a1..a12231fe 100644 --- a/docs/sync.md +++ b/docs/sync.md @@ -10,6 +10,8 @@ - [2. Download & run](#2-download--run) - [Examples](#examples) - [Github.com](#githubcom) + - [GitHub Enterprise Server](#github-enterprise-server) + - [GitHub Enterprise Cloud](#github-enterprise-cloud) - [Known limitations](#known-limitations) ## Prerequisites @@ -60,6 +62,26 @@ In dry-run mode: Live mode: `DEBUG=*snyk* SNYK_TOKEN=xxxx snyk-api-import sync --orgPublicId= --source=github` + +### GitHub Enterprise Server + +In dry-run mode: +`DEBUG=*snyk* SNYK_TOKEN=xxxx snyk-api-import sync --orgPublicId= --source=github-enterprise --sourceUrl=https://custom.ghe.com --dryRun=true` + +Live mode: +`DEBUG=*snyk* SNYK_TOKEN=xxxx snyk-api-import sync --orgPublicId= --source=github-enterprise --sourceUrl=https://custom.ghe.com` + + + +### GitHub Enterprise Cloud + +In dry-run mode: +`DEBUG=*snyk* SNYK_TOKEN=xxxx snyk-api-import sync --orgPublicId= --source=github-enterprise --dryRun=true` + +Live mode: +`DEBUG=*snyk* SNYK_TOKEN=xxxx snyk-api-import sync --orgPublicId= --source=github-enterprise` + + # Known limitations - Any organizations using a custom branch feature are currently not supported, `sync` will not continue. - ANy organizations that previously used the custom feature flag should ideally delete all existing projects & re-import to restore the project names to standard format (do not include a branch in the project name). `sync` will work regardless but may cause confusion as the project name will reference a branch that is not likely to be the actual branch being tested. diff --git a/src/cmds/sync.ts b/src/cmds/sync.ts index 0b83c825..bec8bdef 100644 --- a/src/cmds/sync.ts +++ b/src/cmds/sync.ts @@ -22,7 +22,6 @@ export const builder = { default: undefined, desc: 'Custom base url for the source API that can list organizations (e.g. Github Enterprise url)', }, - // TODO: needs integration Type for GHE<> Github setup source: { required: true, default: SupportedIntegrationTypesUpdateProject.GITHUB, diff --git a/src/lib/types.ts b/src/lib/types.ts index 014b8bfb..65188357 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -93,6 +93,7 @@ export enum SupportedIntegrationTypesImportOrgData { export enum SupportedIntegrationTypesUpdateProject { GITHUB = 'github', + GHE = 'github-enterprise', } // used to generate imported targets that exist in Snyk @@ -107,12 +108,6 @@ export enum SupportedIntegrationTypesToListSnykTargets { AZURE_REPOS = 'azure-repos', BITBUCKET_SERVER = 'bitbucket-server', } -interface ImportingUser { - id: string; - name: string; - username: string; - email: string; -} export interface CommandResult { fileName: string | undefined; diff --git a/src/scripts/sync/sync-org-projects.ts b/src/scripts/sync/sync-org-projects.ts index 78c62620..4e1f5888 100644 --- a/src/scripts/sync/sync-org-projects.ts +++ b/src/scripts/sync/sync-org-projects.ts @@ -26,6 +26,7 @@ export function isSourceConfigured( ): () => void { const getDefaultBranchGenerators = { [SupportedIntegrationTypesUpdateProject.GITHUB]: isGithubConfigured, + [SupportedIntegrationTypesUpdateProject.GHE]: isGithubConfigured, }; return getDefaultBranchGenerators[origin]; } @@ -34,7 +35,7 @@ export async function updateOrgTargets( publicOrgId: string, sources: SupportedIntegrationTypesUpdateProject[], dryRun = false, - host?: string, + sourceUrl?: string, ): Promise<{ fileName: string; failedFileName: string; @@ -124,7 +125,7 @@ export async function updateOrgTargets( publicOrgId, targets, dryRun, - host, + sourceUrl, ); res.processedTargets += response.processedTargets; res.meta.projects.updated.push(...response.meta.projects.updated); @@ -156,7 +157,7 @@ export async function updateTargets( orgId: string, targets: SnykTarget[], dryRun = false, - host?: string, + sourceUrl?: string, ): Promise<{ processedTargets: number; meta: { @@ -181,7 +182,7 @@ export async function updateTargets( orgId, target, dryRun, - host, + sourceUrl, ); updatedProjects.push(...updated); failedProjects.push(...failed); diff --git a/src/scripts/sync/sync-projects-per-target.ts b/src/scripts/sync/sync-projects-per-target.ts index d2a6359c..f2f5d24e 100644 --- a/src/scripts/sync/sync-projects-per-target.ts +++ b/src/scripts/sync/sync-projects-per-target.ts @@ -16,6 +16,7 @@ export function getBranchGenerator( const getDefaultBranchGenerators = { [SupportedIntegrationTypesUpdateProject.GITHUB]: getGithubReposDefaultBranch, + [SupportedIntegrationTypesUpdateProject.GHE]: getGithubReposDefaultBranch, }; return getDefaultBranchGenerators[origin]; } @@ -72,7 +73,7 @@ export async function bulkUpdateProjectsBranch( orgId: string, projects: SnykProject[], dryRun = false, - host?: string, + sourceUrl?: string, ): Promise<{ updated: ProjectUpdate[]; failed: ProjectUpdateFailure[] }> { const updatedProjects: ProjectUpdate[] = []; const failedProjects: ProjectUpdateFailure[] = []; @@ -82,7 +83,7 @@ export async function bulkUpdateProjectsBranch( try { const target = targetGenerators[origin](projects[0]); debug(`Getting default branch via ${origin} for ${projects[0].name}`); - defaultBranch = await getBranchGenerator(origin)(target, host); + defaultBranch = await getBranchGenerator(origin)(target, sourceUrl); } catch (e) { debug(e); const error = `Getting default branch via ${origin} API failed with error: ${e.message}`; diff --git a/test/scripts/sync/sync-org-projects.test.ts b/test/scripts/sync/sync-org-projects.test.ts index f8486e29..9373fee6 100644 --- a/test/scripts/sync/sync-org-projects.test.ts +++ b/test/scripts/sync/sync-org-projects.test.ts @@ -28,11 +28,13 @@ describe('updateTargets', () => { userAgentPrefix: 'snyk-api-import:tests', }); let githubSpy: jest.SpyInstance; - let projectsSpy: jest.SpyInstance; + let updateProjectsSpy: jest.SpyInstance; + let listProjectsSpy: jest.SpyInstance; beforeAll(() => { githubSpy = jest.spyOn(github, 'getGithubReposDefaultBranch'); - projectsSpy = jest.spyOn(projectApi, 'updateProject'); + updateProjectsSpy = jest.spyOn(projectApi, 'updateProject'); + listProjectsSpy = jest.spyOn(lib, 'listProjects'); }, 1000); afterAll(async () => { @@ -107,7 +109,7 @@ describe('updateTargets', () => { .spyOn(lib, 'listProjects') .mockImplementation(() => Promise.resolve(projectsAPIResponse)); githubSpy.mockImplementation(() => Promise.resolve(defaultBranch)); - projectsSpy.mockImplementation(() => + updateProjectsSpy.mockImplementation(() => Promise.resolve({ ...projectsAPIResponse, branch: defaultBranch }), ); // Act @@ -170,11 +172,11 @@ describe('updateTargets', () => { const defaultBranch = projectsAPIResponse.projects[0].branch; - jest - .spyOn(lib, 'listProjects') - .mockImplementation(() => Promise.resolve(projectsAPIResponse)); + listProjectsSpy.mockImplementation(() => + Promise.resolve(projectsAPIResponse), + ); githubSpy.mockImplementation(() => Promise.resolve(defaultBranch)); - projectsSpy.mockImplementation(() => + updateProjectsSpy.mockImplementation(() => Promise.resolve({ ...projectsAPIResponse, branch: defaultBranch }), ); @@ -264,11 +266,11 @@ describe('updateTargets', () => { }, ]; - jest - .spyOn(lib, 'listProjects') - .mockImplementation(() => Promise.resolve(projectsAPIResponse)); + listProjectsSpy.mockImplementation(() => + Promise.resolve(projectsAPIResponse), + ); githubSpy.mockImplementation(() => Promise.resolve(defaultBranch)); - projectsSpy + updateProjectsSpy .mockImplementationOnce(() => Promise.resolve({ ...projectsAPIResponse, branch: defaultBranch }), ) @@ -290,6 +292,119 @@ describe('updateTargets', () => { }); }, 5000); }); + describe('Github Enterprise', () => { + it('updates several projects from the same target 1 failed 1 success', async () => { + // Arrange + const testTargets = [ + { + attributes: { + displayName: 'snyk/monorepo', + isPrivate: false, + origin: 'github-enterprise', + remoteUrl: null, + }, + id: 'af137b96-6966-46c1-826b-2e79ac49bbxx', + relationships: { + org: { + data: { + id: 'af137b96-6966-46c1-826b-2e79ac49bbxx', + type: 'org', + }, + links: {}, + meta: {}, + }, + }, + type: 'target', + }, + ]; + const orgId = 'af137b96-6966-46c1-826b-2e79ac49bbxx'; + const projectsAPIResponse: ProjectsResponse = { + org: { + id: orgId, + }, + projects: [ + { + name: 'snyk/monorepo:build.gradle', + id: '3626066d-21a7-424f-b6fc-dc0d222d8e4a', + created: '2018-10-29T09:50:54.014Z', + origin: 'github-enterprise', + type: 'npm', + branch: 'master', + }, + { + name: 'snyk/monorepo(main):package.json', + id: 'f57afea5-8fed-41d8-a8fd-d374c0944b07', + created: '2018-10-29T09:50:54.014Z', + origin: 'github-enterprise', + type: 'maven', + branch: 'master', + }, + ], + }; + + const defaultBranch = 'develop'; + const updated: syncProjectsForTarget.ProjectUpdate[] = [ + { + projectPublicId: projectsAPIResponse.projects[0].id, + from: projectsAPIResponse.projects[0].branch!, + to: defaultBranch, + type: 'branch', + dryRun: false, + }, + ]; + const failed: syncProjectsForTarget.ProjectUpdateFailure[] = [ + { + errorMessage: + 'Failed to update project f57afea5-8fed-41d8-a8fd-d374c0944b07 via Snyk API. ERROR: Error', + projectPublicId: projectsAPIResponse.projects[1].id, + from: projectsAPIResponse.projects[1].branch!, + to: defaultBranch, + type: 'branch', + dryRun: false, + }, + ]; + + listProjectsSpy.mockImplementation(() => + Promise.resolve(projectsAPIResponse), + ); + githubSpy.mockImplementation(() => Promise.resolve(defaultBranch)); + updateProjectsSpy + .mockImplementationOnce(() => + Promise.resolve({ ...projectsAPIResponse, branch: defaultBranch }), + ) + .mockImplementationOnce(() => + Promise.reject({ statusCode: '404', message: 'Error' }), + ); + // Act + const res = await updateTargets( + requestManager, + orgId, + testTargets, + false, + 'https://custom-ghe.com', + ); + + // Assert + expect(res).toStrictEqual({ + processedTargets: 1, + meta: { + projects: { + updated: updated.map((u) => ({ ...u, target: testTargets[0] })), + failed: failed.map((f) => ({ ...f, target: testTargets[0] })), + }, + }, + }); + + expect(githubSpy).toBeCalledWith( + { + branch: 'master', + name: 'monorepo', + owner: 'snyk', + }, + 'https://custom-ghe.com', + ); + }, 5000); + }); }); describe('updateOrgTargets', () => { const OLD_ENV = process.env; @@ -323,7 +438,7 @@ describe('updateOrgTargets', () => { jest.clearAllMocks(); }); - describe('Github', () => { + describe('Errors', () => { it('throws if only unsupported origins requested', async () => { await expect( updateOrgTargets('xxx', ['unsupported' as any]), @@ -379,9 +494,54 @@ describe('updateOrgTargets', () => { processedTargets: 0, }); }); - it.todo('github is not configured'); + }); + + describe('Github', () => { + it('github is not configured', async () => { + // Arrange + delete process.env.GITHUB_TOKEN; + const targets: SnykTarget[] = [ + { + attributes: { + displayName: 'foo/bar', + isPrivate: true, + origin: 'github', + remoteUrl: null, + }, + id: 'xxx', + relationships: {} as unknown as SnykTargetRelationships, + type: 'target', + }, + ]; + const projects: SnykProject[] = [ + { + name: 'example', + id: '123', + created: 'date', + origin: 'github', + type: 'npm', + branch: 'main', + }, + ]; + featureFlagsSpy.mockResolvedValue(false); + listTargetsSpy.mockResolvedValue({ targets }); + listProjectsSpy.mockRejectedValue(projects); + logUpdatedProjectsSpy.mockResolvedValue(null); + + // Act + await expect(() => + updateOrgTargets('xxx', [ + SupportedIntegrationTypesUpdateProject.GITHUB, + ]), + ).rejects.toThrowError( + "Please set the GITHUB_TOKEN e.g. export GITHUB_TOKEN='mypersonalaccesstoken123'", + ); + + // Assert + }); it.todo('skips extra unsupported source, but finishes supported'); it('skips target & projects error if getting default branch fails', async () => { + // Arrange const targets: SnykTarget[] = [ { attributes: { @@ -410,9 +570,11 @@ describe('updateOrgTargets', () => { listProjectsSpy.mockRejectedValue(projects); logUpdatedProjectsSpy.mockResolvedValue(null); + // Act const res = await updateOrgTargets('xxx', [ SupportedIntegrationTypesUpdateProject.GITHUB, ]); + expect(res).toStrictEqual({ failedFileName: expect.stringMatching('/failed-to-update-projects.log'), fileName: expect.stringMatching('/updated-projects.log'), @@ -426,7 +588,8 @@ describe('updateOrgTargets', () => { }); }); - it('Successfully updated several targets (dryRun mode)', async () => { + it('successfully updated several targets (dryRun mode)', async () => { + // Arrange const targets: SnykTarget[] = [ { attributes: { @@ -528,4 +691,148 @@ describe('updateOrgTargets', () => { expect(logUpdatedProjectsSpy).toHaveBeenCalledTimes(2); }); }); + + describe('Github Enterprise', () => { + it('github enterprise is not configured', async () => { + // Arrange + delete process.env.GITHUB_TOKEN; + const targets: SnykTarget[] = [ + { + attributes: { + displayName: 'foo/bar', + isPrivate: true, + origin: 'github-enterprise', + remoteUrl: null, + }, + id: 'xxx', + relationships: {} as unknown as SnykTargetRelationships, + type: 'target', + }, + ]; + const projects: SnykProject[] = [ + { + name: 'example', + id: '123', + created: 'date', + origin: 'github-enterprise', + type: 'npm', + branch: 'main', + }, + ]; + featureFlagsSpy.mockResolvedValue(false); + listTargetsSpy.mockResolvedValue({ targets }); + listProjectsSpy.mockRejectedValue(projects); + logUpdatedProjectsSpy.mockResolvedValue(null); + + // Act & Assert + await expect(() => + updateOrgTargets('xxx', [SupportedIntegrationTypesUpdateProject.GHE]), + ).rejects.toThrowError( + "Please set the GITHUB_TOKEN e.g. export GITHUB_TOKEN='mypersonalaccesstoken123'", + ); + }); + it('successfully updated several targets (dryRun mode)', async () => { + // Arrange + const targets: SnykTarget[] = [ + { + attributes: { + displayName: 'snyk/bar', + isPrivate: true, + origin: 'github-enterprise', + remoteUrl: null, + }, + id: uuid.v4(), + relationships: {} as unknown as SnykTargetRelationships, + type: 'target', + }, + { + attributes: { + displayName: 'snyk/foo', + isPrivate: false, + origin: 'github-enterprise', + remoteUrl: null, + }, + id: uuid.v4(), + relationships: {} as unknown as SnykTargetRelationships, + type: 'target', + }, + ]; + const updatedProjectId1 = uuid.v4(); + const updatedProjectId2 = uuid.v4(); + const projectsTarget1: SnykProject[] = [ + { + name: 'snyk/bar', + id: updatedProjectId1, + created: 'date', + origin: 'github', + type: 'npm', + branch: 'main', + }, + ]; + const projectsTarget2: SnykProject[] = [ + { + name: 'snyk/foo', + id: updatedProjectId2, + created: 'date', + origin: 'github-enterprise', + type: 'yarn', + branch: 'develop', + }, + ]; + featureFlagsSpy.mockResolvedValueOnce(false); + listTargetsSpy.mockResolvedValueOnce({ targets }); + listProjectsSpy + .mockResolvedValueOnce({ projects: projectsTarget1 }) + .mockResolvedValueOnce({ projects: projectsTarget2 }); + + logUpdatedProjectsSpy.mockResolvedValueOnce(null); + const defaultBranch = 'new-branch'; + githubSpy.mockResolvedValue(defaultBranch); + const updated: syncProjectsForTarget.ProjectUpdate[] = [ + { + projectPublicId: updatedProjectId1, + from: projectsTarget1[0].branch!, + to: defaultBranch, + type: 'branch', + dryRun: true, + target: targets[0], + }, + { + projectPublicId: updatedProjectId2, + from: projectsTarget2[0].branch!, + to: defaultBranch, + type: 'branch', + dryRun: true, + target: targets[1], + }, + ]; + const failed: syncProjectsForTarget.ProjectUpdateFailure[] = []; + + // Act + const res = await updateOrgTargets( + 'xxx', + [SupportedIntegrationTypesUpdateProject.GHE], + true, + 'https://custom.ghe.com', + ); + // Assert + expect(res).toStrictEqual({ + failedFileName: expect.stringMatching('/failed-to-update-projects.log'), + fileName: expect.stringMatching('/updated-projects.log'), + meta: { + projects: { + updated, + failed, + }, + }, + processedTargets: 2, + }); + expect(featureFlagsSpy).toHaveBeenCalledTimes(1); + expect(listTargetsSpy).toHaveBeenCalledTimes(1); + expect(listProjectsSpy).toHaveBeenCalledTimes(2); + expect(githubSpy).toBeCalledTimes(2); + expect(updateProjectSpy).not.toHaveBeenCalled(); + expect(logUpdatedProjectsSpy).toHaveBeenCalledTimes(2); + }); + }); }); diff --git a/test/system/__snapshots__/sync.test.ts.snap b/test/system/__snapshots__/sync.test.ts.snap index 4017e3c4..d77ee7e4 100644 --- a/test/system/__snapshots__/sync.test.ts.snap +++ b/test/system/__snapshots__/sync.test.ts.snap @@ -16,7 +16,7 @@ Options: (e.g. Github Enterprise url) --source List of sources to be synced e.g. Github, Github Enterprise, Gitlab, Bitbucket Server, Bitbucket Cloud - [required] [choices: \\"github\\"] [default: \\"github\\"] + [required] [choices: \\"github\\", \\"github-enterprise\\"] [default: \\"github\\"] --dryRun Dry run option. Will create a log file listing the potential updates [default: false]" `; diff --git a/test/system/sync.test.ts b/test/system/sync.test.ts index 54650fd2..89d5ddd5 100644 --- a/test/system/sync.test.ts +++ b/test/system/sync.test.ts @@ -25,7 +25,7 @@ describe('`snyk-api-import help <...>`', () => { }); }); - it('Fails when given a bad/non-existent org ID `123456789`', (done) => { + it('fails when given a bad/non-existent org ID `123456789`', (done) => { const logPath = path.resolve(__dirname + '/fixtures'); exec( @@ -58,7 +58,7 @@ describe('`snyk-api-import help <...>`', () => { }); }, 40000); - it('Throws an error for an unsupported SCM like Bitbucket Server', (done) => { + it('throws an error for an unsupported SCM like Bitbucket Server', (done) => { const logPath = path.resolve(__dirname); exec( @@ -72,55 +72,10 @@ describe('`snyk-api-import help <...>`', () => { }, }, (err, stdout, stderr) => { - expect(stderr).toMatchInlineSnapshot(` - "index.js sync - - Sync targets (e.g. repos) and their projects between Snyk and SCM for a given - organization. Actions include: - - updating monitored branch in Snyk to match the default branch from SCM - - Options: - --version Show version number [boolean] - --help Show help [boolean] - --orgPublicId Public id of the organization in Snyk that will be updated - [required] - --sourceUrl Custom base url for the source API that can list organizations - (e.g. Github Enterprise url) - --source List of sources to be synced e.g. Github, Github Enterprise, - Gitlab, Bitbucket Server, Bitbucket Cloud - [required] [choices: \\"github\\"] [default: \\"github\\"] - --dryRun Dry run option. Will create a log file listing the potential - updates [default: false] - - Invalid values: - Argument: source, Given: \\"bitbucket-server\\", Choices: \\"github\\" - " - `); - expect(err!.message).toMatchInlineSnapshot(` - "Command failed: node ./dist/index.js sync --orgPublicId=123456789 --source=bitbucket-server --sourceUrl=somewhere.com - index.js sync - - Sync targets (e.g. repos) and their projects between Snyk and SCM for a given - organization. Actions include: - - updating monitored branch in Snyk to match the default branch from SCM - - Options: - --version Show version number [boolean] - --help Show help [boolean] - --orgPublicId Public id of the organization in Snyk that will be updated - [required] - --sourceUrl Custom base url for the source API that can list organizations - (e.g. Github Enterprise url) - --source List of sources to be synced e.g. Github, Github Enterprise, - Gitlab, Bitbucket Server, Bitbucket Cloud - [required] [choices: \\"github\\"] [default: \\"github\\"] - --dryRun Dry run option. Will create a log file listing the potential - updates [default: false] - - Invalid values: - Argument: source, Given: \\"bitbucket-server\\", Choices: \\"github\\" - " - `); + expect(stderr).toMatch(`Argument: source, Given: "bitbucket-server"`); + expect(err!.message).toMatch( + `Argument: source, Given: "bitbucket-server"`, + ); expect(stdout).toEqual(''); }, ).on('exit', (code) => {