From 3be8c93277e793fe1248dfc6a1e530f398aba1c4 Mon Sep 17 00:00:00 2001 From: Michal Dorner Date: Mon, 8 Mar 2021 15:09:07 +0100 Subject: [PATCH 1/7] Fix fetching git history + fallback to unshallow repo --- src/git.ts | 83 ++++++++++++++++++++++++++--------------------------- src/main.ts | 10 +++---- 2 files changed, 46 insertions(+), 47 deletions(-) diff --git a/src/git.ts b/src/git.ts index f34a5cce..08953138 100644 --- a/src/git.ts +++ b/src/git.ts @@ -18,20 +18,20 @@ export async function getChangesInLastCommit(): Promise { return parseGitDiffOutput(output) } -export async function getChanges(ref: string): Promise { - if (!(await hasCommit(ref))) { +export async function getChanges(baseRef: string): Promise { + if (!(await hasCommit(baseRef))) { // Fetch single commit - core.startGroup(`Fetching ${ref} from origin`) - await exec('git', ['fetch', '--depth=1', '--no-tags', 'origin', ref]) + core.startGroup(`Fetching ${baseRef} from origin`) + await exec('git', ['fetch', '--depth=1', '--no-tags', 'origin', baseRef]) core.endGroup() } // Get differences between ref and HEAD - core.startGroup(`Change detection ${ref}..HEAD`) + core.startGroup(`Change detection ${baseRef}..HEAD`) let output = '' try { // Two dots '..' change detection - directly compares two versions - output = (await exec('git', ['diff', '--no-renames', '--name-status', '-z', `${ref}..HEAD`])).stdout + output = (await exec('git', ['diff', '--no-renames', '--name-status', '-z', `${baseRef}..HEAD`])).stdout } finally { fixStdOutNullTermination() core.endGroup() @@ -54,50 +54,49 @@ export async function getChangesOnHead(): Promise { return parseGitDiffOutput(output) } -export async function getChangesSinceMergeBase(ref: string, initialFetchDepth: number): Promise { - if (!(await hasCommit(ref))) { - // Fetch and add base branch - core.startGroup(`Fetching ${ref}`) - try { - await exec('git', ['fetch', `--depth=${initialFetchDepth}`, '--no-tags', 'origin', `${ref}:${ref}`]) - } finally { - core.endGroup() - } - } - +export async function getChangesSinceMergeBase( + baseRef: string, + ref: string, + initialFetchDepth: number +): Promise { async function hasMergeBase(): Promise { - return (await exec('git', ['merge-base', ref, 'HEAD'], {ignoreReturnCode: true})).code === 0 + return (await exec('git', ['merge-base', baseRef, ref], {ignoreReturnCode: true})).code === 0 } - async function countCommits(): Promise { - return (await getNumberOfCommits('HEAD')) + (await getNumberOfCommits(ref)) + let noMergeBase = false + core.startGroup(`Searching for merge-base ${baseRef}...${ref}`) + try { + let lastCommitCount = await getCommitCount() + let depth = Math.max(lastCommitCount * 2, initialFetchDepth) + while (!(await hasMergeBase())) { + await exec('git', ['fetch', `--depth=${depth}`, 'origin', `${baseRef}:${baseRef}`, `${ref}:${ref}`]) + const commitCount = await getCommitCount() + if (commitCount === lastCommitCount) { + core.info('No more commits were fetched') + core.info('Last attempt will be to fetch full history') + await exec('git', ['fetch', '--unshallow']) + if (!(await hasMergeBase())) { + noMergeBase = true + } + break + } + depth = Math.min(depth * 2, Number.MAX_SAFE_INTEGER) + } + } finally { + core.endGroup() } - core.startGroup(`Searching for merge-base with ${ref}`) - // Fetch more commits until merge-base is found - if (!(await hasMergeBase())) { - let deepen = initialFetchDepth - let lastCommitsCount = await countCommits() - do { - await exec('git', ['fetch', `--deepen=${deepen}`, '--no-tags']) - const count = await countCommits() - if (count <= lastCommitsCount) { - core.info('No merge base found - all files will be listed as added') - core.endGroup() - return await listAllFilesAsAdded() - } - lastCommitsCount = count - deepen = Math.min(deepen * 2, Number.MAX_SAFE_INTEGER) - } while (!(await hasMergeBase())) + if (noMergeBase) { + core.warning('No merge base found - all files will be listed as added') + return await listAllFilesAsAdded() } - core.endGroup() // Get changes introduced on HEAD compared to ref - core.startGroup(`Change detection ${ref}...HEAD`) + core.startGroup(`Change detection ${baseRef}...${ref}`) let output = '' try { // Three dots '...' change detection - finds merge-base and compares against it - output = (await exec('git', ['diff', '--no-renames', '--name-status', '-z', `${ref}...HEAD`])).stdout + output = (await exec('git', ['diff', '--no-renames', '--name-status', '-z', `${baseRef}...${ref}`])).stdout } finally { fixStdOutNullTermination() core.endGroup() @@ -150,7 +149,7 @@ export async function getCurrentRef(): Promise { return describe.stdout.trim() } - return (await exec('git', ['rev-parse', 'HEAD'])).stdout.trim() + return (await exec('git', ['rev-parse', HEAD])).stdout.trim() } finally { core.endGroup() } @@ -181,8 +180,8 @@ async function hasCommit(ref: string): Promise { } } -async function getNumberOfCommits(ref: string): Promise { - const output = (await exec('git', ['rev-list', `--count`, ref])).stdout +async function getCommitCount(): Promise { + const output = (await exec('git', ['rev-list', '--count', '--all'])).stdout const count = parseInt(output) return isNaN(count) ? 0 : count } diff --git a/src/main.ts b/src/main.ts index a54776f1..395ab69f 100644 --- a/src/main.ts +++ b/src/main.ts @@ -80,7 +80,7 @@ async function getChangedFilesFromGit(base: string, initialFetchDepth: number): const beforeSha = github.context.eventName === 'push' ? (github.context.payload as Webhooks.WebhookPayloadPush).before : null - const pushRef = + const ref = git.getShortName(github.context.ref) || (core.warning(`'ref' field is missing in event payload - using current branch, tag or commit SHA`), await git.getCurrentRef()) @@ -93,11 +93,11 @@ async function getChangedFilesFromGit(base: string, initialFetchDepth: number): } const isBaseRefSha = git.isGitSha(baseRef) - const isBaseSameAsPush = baseRef === pushRef + const isBaseRefSameAsRef = baseRef === ref // If base is commit SHA we will do comparison against the referenced commit // Or if base references same branch it was pushed to, we will do comparison against the previously pushed commit - if (isBaseRefSha || isBaseSameAsPush) { + if (isBaseRefSha || isBaseRefSameAsRef) { if (!isBaseRefSha && !beforeSha) { core.warning(`'before' field is missing in event payload - changes will be detected from last commit`) return await git.getChangesInLastCommit() @@ -109,7 +109,7 @@ async function getChangedFilesFromGit(base: string, initialFetchDepth: number): if (baseSha === git.NULL_SHA) { if (defaultRef && baseRef !== defaultRef) { core.info(`First push of a branch detected - changes will be detected against the default branch ${defaultRef}`) - return await git.getChangesSinceMergeBase(defaultRef, initialFetchDepth) + return await git.getChangesSinceMergeBase(defaultRef, ref, initialFetchDepth) } else { core.info('Initial push detected - all files will be listed as added') return await git.listAllFilesAsAdded() @@ -122,7 +122,7 @@ async function getChangedFilesFromGit(base: string, initialFetchDepth: number): // Changes introduced by current branch against the base branch core.info(`Changes will be detected against the branch ${baseRef}`) - return await git.getChangesSinceMergeBase(baseRef, initialFetchDepth) + return await git.getChangesSinceMergeBase(baseRef, ref, initialFetchDepth) } // Uses github REST api to get list of files changed in PR From 31c576896e759c70a84fb70a7598f3d2c1426737 Mon Sep 17 00:00:00 2001 From: Michal Dorner Date: Mon, 8 Mar 2021 15:12:58 +0100 Subject: [PATCH 2/7] Fix lastCommitCount has not been updated --- dist/index.js | 93 +++++++++++++++++++++++++-------------------------- src/git.ts | 1 + 2 files changed, 46 insertions(+), 48 deletions(-) diff --git a/dist/index.js b/dist/index.js index fcf0faae..815793f9 100644 --- a/dist/index.js +++ b/dist/index.js @@ -3830,19 +3830,19 @@ async function getChangesInLastCommit() { return parseGitDiffOutput(output); } exports.getChangesInLastCommit = getChangesInLastCommit; -async function getChanges(ref) { - if (!(await hasCommit(ref))) { +async function getChanges(baseRef) { + if (!(await hasCommit(baseRef))) { // Fetch single commit - core.startGroup(`Fetching ${ref} from origin`); - await exec_1.default('git', ['fetch', '--depth=1', '--no-tags', 'origin', ref]); + core.startGroup(`Fetching ${baseRef} from origin`); + await exec_1.default('git', ['fetch', '--depth=1', '--no-tags', 'origin', baseRef]); core.endGroup(); } // Get differences between ref and HEAD - core.startGroup(`Change detection ${ref}..HEAD`); + core.startGroup(`Change detection ${baseRef}..HEAD`); let output = ''; try { // Two dots '..' change detection - directly compares two versions - output = (await exec_1.default('git', ['diff', '--no-renames', '--name-status', '-z', `${ref}..HEAD`])).stdout; + output = (await exec_1.default('git', ['diff', '--no-renames', '--name-status', '-z', `${baseRef}..HEAD`])).stdout; } finally { fixStdOutNullTermination(); @@ -3865,47 +3865,44 @@ async function getChangesOnHead() { return parseGitDiffOutput(output); } exports.getChangesOnHead = getChangesOnHead; -async function getChangesSinceMergeBase(ref, initialFetchDepth) { - if (!(await hasCommit(ref))) { - // Fetch and add base branch - core.startGroup(`Fetching ${ref}`); - try { - await exec_1.default('git', ['fetch', `--depth=${initialFetchDepth}`, '--no-tags', 'origin', `${ref}:${ref}`]); - } - finally { - core.endGroup(); - } - } +async function getChangesSinceMergeBase(baseRef, ref, initialFetchDepth) { async function hasMergeBase() { - return (await exec_1.default('git', ['merge-base', ref, 'HEAD'], { ignoreReturnCode: true })).code === 0; - } - async function countCommits() { - return (await getNumberOfCommits('HEAD')) + (await getNumberOfCommits(ref)); - } - core.startGroup(`Searching for merge-base with ${ref}`); - // Fetch more commits until merge-base is found - if (!(await hasMergeBase())) { - let deepen = initialFetchDepth; - let lastCommitsCount = await countCommits(); - do { - await exec_1.default('git', ['fetch', `--deepen=${deepen}`, '--no-tags']); - const count = await countCommits(); - if (count <= lastCommitsCount) { - core.info('No merge base found - all files will be listed as added'); - core.endGroup(); - return await listAllFilesAsAdded(); + return (await exec_1.default('git', ['merge-base', baseRef, ref], { ignoreReturnCode: true })).code === 0; + } + let noMergeBase = false; + core.startGroup(`Searching for merge-base ${baseRef}...${ref}`); + try { + let lastCommitCount = await getCommitCount(); + let depth = Math.max(lastCommitCount * 2, initialFetchDepth); + while (!(await hasMergeBase())) { + await exec_1.default('git', ['fetch', `--depth=${depth}`, 'origin', `${baseRef}:${baseRef}`, `${ref}:${ref}`]); + const commitCount = await getCommitCount(); + if (commitCount === lastCommitCount) { + core.info('No more commits were fetched'); + core.info('Last attempt will be to fetch full history'); + await exec_1.default('git', ['fetch', '--unshallow']); + if (!(await hasMergeBase())) { + noMergeBase = true; + } + break; } - lastCommitsCount = count; - deepen = Math.min(deepen * 2, Number.MAX_SAFE_INTEGER); - } while (!(await hasMergeBase())); + depth = Math.min(depth * 2, Number.MAX_SAFE_INTEGER); + lastCommitCount = commitCount; + } + } + finally { + core.endGroup(); + } + if (noMergeBase) { + core.warning('No merge base found - all files will be listed as added'); + return await listAllFilesAsAdded(); } - core.endGroup(); // Get changes introduced on HEAD compared to ref - core.startGroup(`Change detection ${ref}...HEAD`); + core.startGroup(`Change detection ${baseRef}...${ref}`); let output = ''; try { // Three dots '...' change detection - finds merge-base and compares against it - output = (await exec_1.default('git', ['diff', '--no-renames', '--name-status', '-z', `${ref}...HEAD`])).stdout; + output = (await exec_1.default('git', ['diff', '--no-renames', '--name-status', '-z', `${baseRef}...${ref}`])).stdout; } finally { fixStdOutNullTermination(); @@ -3956,7 +3953,7 @@ async function getCurrentRef() { if (describe.code === 0) { return describe.stdout.trim(); } - return (await exec_1.default('git', ['rev-parse', 'HEAD'])).stdout.trim(); + return (await exec_1.default('git', ['rev-parse', exports.HEAD])).stdout.trim(); } finally { core.endGroup(); @@ -3988,8 +3985,8 @@ async function hasCommit(ref) { core.endGroup(); } } -async function getNumberOfCommits(ref) { - const output = (await exec_1.default('git', ['rev-list', `--count`, ref])).stdout; +async function getCommitCount() { + const output = (await exec_1.default('git', ['rev-list', '--count', '--all'])).stdout; const count = parseInt(output); return isNaN(count) ? 0 : count; } @@ -4709,7 +4706,7 @@ async function getChangedFilesFromGit(base, initialFetchDepth) { var _a; const defaultRef = (_a = github.context.payload.repository) === null || _a === void 0 ? void 0 : _a.default_branch; const beforeSha = github.context.eventName === 'push' ? github.context.payload.before : null; - const pushRef = git.getShortName(github.context.ref) || + const ref = git.getShortName(github.context.ref) || (core.warning(`'ref' field is missing in event payload - using current branch, tag or commit SHA`), await git.getCurrentRef()); const baseRef = git.getShortName(base) || defaultRef; @@ -4717,10 +4714,10 @@ async function getChangedFilesFromGit(base, initialFetchDepth) { throw new Error("This action requires 'base' input to be configured or 'repository.default_branch' to be set in the event payload"); } const isBaseRefSha = git.isGitSha(baseRef); - const isBaseSameAsPush = baseRef === pushRef; + const isBaseRefSameAsRef = baseRef === ref; // If base is commit SHA we will do comparison against the referenced commit // Or if base references same branch it was pushed to, we will do comparison against the previously pushed commit - if (isBaseRefSha || isBaseSameAsPush) { + if (isBaseRefSha || isBaseRefSameAsRef) { if (!isBaseRefSha && !beforeSha) { core.warning(`'before' field is missing in event payload - changes will be detected from last commit`); return await git.getChangesInLastCommit(); @@ -4731,7 +4728,7 @@ async function getChangedFilesFromGit(base, initialFetchDepth) { if (baseSha === git.NULL_SHA) { if (defaultRef && baseRef !== defaultRef) { core.info(`First push of a branch detected - changes will be detected against the default branch ${defaultRef}`); - return await git.getChangesSinceMergeBase(defaultRef, initialFetchDepth); + return await git.getChangesSinceMergeBase(defaultRef, ref, initialFetchDepth); } else { core.info('Initial push detected - all files will be listed as added'); @@ -4743,7 +4740,7 @@ async function getChangedFilesFromGit(base, initialFetchDepth) { } // Changes introduced by current branch against the base branch core.info(`Changes will be detected against the branch ${baseRef}`); - return await git.getChangesSinceMergeBase(baseRef, initialFetchDepth); + return await git.getChangesSinceMergeBase(baseRef, ref, initialFetchDepth); } // Uses github REST api to get list of files changed in PR async function getChangedFilesFromApi(token, pullRequest) { diff --git a/src/git.ts b/src/git.ts index 08953138..2cfa8aeb 100644 --- a/src/git.ts +++ b/src/git.ts @@ -81,6 +81,7 @@ export async function getChangesSinceMergeBase( break } depth = Math.min(depth * 2, Number.MAX_SAFE_INTEGER) + lastCommitCount = commitCount } } finally { core.endGroup() From 68792bf56a01ea3d5043c4a50063dc62185a03a6 Mon Sep 17 00:00:00 2001 From: Michal Dorner Date: Mon, 8 Mar 2021 15:24:06 +0100 Subject: [PATCH 3/7] Allow single line filters --- dist/index.js | 2 +- src/main.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dist/index.js b/dist/index.js index 815793f9..da243b57 100644 --- a/dist/index.js +++ b/dist/index.js @@ -4673,7 +4673,7 @@ async function run() { } } function isPathInput(text) { - return !text.includes('\n'); + return !(text.includes('\n') || text.includes(':')); } function getConfigFileContent(configPath) { if (!fs.existsSync(configPath)) { diff --git a/src/main.ts b/src/main.ts index 395ab69f..e3d6ebc4 100644 --- a/src/main.ts +++ b/src/main.ts @@ -40,7 +40,7 @@ async function run(): Promise { } function isPathInput(text: string): boolean { - return !text.includes('\n') + return !(text.includes('\n') || text.includes(':')) } function getConfigFileContent(configPath: string): string { From 8801c887e9ff7fd83cc246b7fce277a3efc4ea70 Mon Sep 17 00:00:00 2001 From: Michal Dorner Date: Mon, 8 Mar 2021 15:29:27 +0100 Subject: [PATCH 4/7] Do not try to update head of current branch --- dist/index.js | 2 +- src/git.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dist/index.js b/dist/index.js index da243b57..a16648d6 100644 --- a/dist/index.js +++ b/dist/index.js @@ -3875,7 +3875,7 @@ async function getChangesSinceMergeBase(baseRef, ref, initialFetchDepth) { let lastCommitCount = await getCommitCount(); let depth = Math.max(lastCommitCount * 2, initialFetchDepth); while (!(await hasMergeBase())) { - await exec_1.default('git', ['fetch', `--depth=${depth}`, 'origin', `${baseRef}:${baseRef}`, `${ref}:${ref}`]); + await exec_1.default('git', ['fetch', `--depth=${depth}`, 'origin', `${baseRef}:${baseRef}`, `${ref}`]); const commitCount = await getCommitCount(); if (commitCount === lastCommitCount) { core.info('No more commits were fetched'); diff --git a/src/git.ts b/src/git.ts index 2cfa8aeb..c58621fd 100644 --- a/src/git.ts +++ b/src/git.ts @@ -69,7 +69,7 @@ export async function getChangesSinceMergeBase( let lastCommitCount = await getCommitCount() let depth = Math.max(lastCommitCount * 2, initialFetchDepth) while (!(await hasMergeBase())) { - await exec('git', ['fetch', `--depth=${depth}`, 'origin', `${baseRef}:${baseRef}`, `${ref}:${ref}`]) + await exec('git', ['fetch', `--depth=${depth}`, 'origin', `${baseRef}:${baseRef}`, `${ref}`]) const commitCount = await getCommitCount() if (commitCount === lastCommitCount) { core.info('No more commits were fetched') From 49abb091ed9a35ee0988a637263db128562e6bf3 Mon Sep 17 00:00:00 2001 From: Michal Dorner Date: Mon, 8 Mar 2021 17:00:52 +0100 Subject: [PATCH 5/7] Use combination of --depth and --deepen --- dist/index.js | 9 ++++++++- src/git.ts | 8 +++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/dist/index.js b/dist/index.js index a16648d6..71893ad6 100644 --- a/dist/index.js +++ b/dist/index.js @@ -3872,10 +3872,17 @@ async function getChangesSinceMergeBase(baseRef, ref, initialFetchDepth) { let noMergeBase = false; core.startGroup(`Searching for merge-base ${baseRef}...${ref}`); try { + let init = true; let lastCommitCount = await getCommitCount(); let depth = Math.max(lastCommitCount * 2, initialFetchDepth); while (!(await hasMergeBase())) { - await exec_1.default('git', ['fetch', `--depth=${depth}`, 'origin', `${baseRef}:${baseRef}`, `${ref}`]); + if (init) { + await exec_1.default('git', ['fetch', `--depth=${depth}`, 'origin', `${baseRef}:${baseRef}`, `${ref}`]); + init = false; + } + else { + await exec_1.default('git', ['fetch', `--deepen=${depth}`, 'origin', baseRef, ref]); + } const commitCount = await getCommitCount(); if (commitCount === lastCommitCount) { core.info('No more commits were fetched'); diff --git a/src/git.ts b/src/git.ts index c58621fd..a55faa6b 100644 --- a/src/git.ts +++ b/src/git.ts @@ -66,10 +66,16 @@ export async function getChangesSinceMergeBase( let noMergeBase = false core.startGroup(`Searching for merge-base ${baseRef}...${ref}`) try { + let init = true let lastCommitCount = await getCommitCount() let depth = Math.max(lastCommitCount * 2, initialFetchDepth) while (!(await hasMergeBase())) { - await exec('git', ['fetch', `--depth=${depth}`, 'origin', `${baseRef}:${baseRef}`, `${ref}`]) + if (init) { + await exec('git', ['fetch', `--depth=${depth}`, 'origin', `${baseRef}:${baseRef}`, `${ref}`]) + init = false + } else { + await exec('git', ['fetch', `--deepen=${depth}`, 'origin', baseRef, ref]) + } const commitCount = await getCommitCount() if (commitCount === lastCommitCount) { core.info('No more commits were fetched') From c90ecaa5a164f8b3a35057976737a185625069d9 Mon Sep 17 00:00:00 2001 From: Michal Dorner Date: Mon, 8 Mar 2021 17:06:12 +0100 Subject: [PATCH 6/7] Increase default value of initial-fetch-depth to 100 For most situation it should be enough to find merge base. Previous value was too slow and overhead of doing fetch was significantly higher than saving of transfer. --- README.md | 2 +- action.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index eaa0bbc2..e78b1a05 100644 --- a/README.md +++ b/README.md @@ -117,7 +117,7 @@ For more information see [CHANGELOG](https://github.com/dorny/paths-filter/blob/ # is found or there are no more commits in the history. # This option takes effect only when changes are detected # using git against base branch (feature branch workflow). - # Default: 20 + # Default: 100 initial-fetch-depth: '' # Enables listing of files matching the filter: diff --git a/action.yml b/action.yml index 894b1149..b25aa5e2 100644 --- a/action.yml +++ b/action.yml @@ -38,7 +38,7 @@ inputs: until the merge-base is found or there are no more commits in the history. This option takes effect only when changes are detected using git against different base branch. required: false - default: '10' + default: '100' outputs: changes: description: JSON array with names of all filters matching any of changed files From 46d2898cef8969688c8b8354591ce041cbe8e745 Mon Sep 17 00:00:00 2001 From: Michal Dorner Date: Mon, 8 Mar 2021 17:10:14 +0100 Subject: [PATCH 7/7] Update CHANGELOG --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 31e166ba..a064e6b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## v2.9.1 +- [Fix fetching git history + fallback to unshallow repo](https://github.com/dorny/paths-filter/pull/74) + ## v2.9.0 - [Add list-files: csv format](https://github.com/dorny/paths-filter/pull/68)