From 9c7b2a2710fd3139816dbcae75996d9580340661 Mon Sep 17 00:00:00 2001 From: Baba Kamyljanov Date: Thu, 5 Sep 2024 12:10:22 +0800 Subject: [PATCH 01/11] added logic to read markdown to get title, uri, isArchived and add these data to history.json --- build-scripts/update-rule-history.ps1 | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/build-scripts/update-rule-history.ps1 b/build-scripts/update-rule-history.ps1 index 7b4cbf010..0b0ce2dd7 100644 --- a/build-scripts/update-rule-history.ps1 +++ b/build-scripts/update-rule-history.ps1 @@ -59,9 +59,35 @@ $historyArray | Foreach-Object { $createdRecord = git log --diff-filter=A --reverse --pretty="%ad%aN%ae" --date=iso-strict -- $_ $createdDetails = $createdRecord -split "" + # Read and parse Markdown file to set title, uri, and archived status + $utf8NoBomEncoding = New-Object System.Text.UTF8Encoding $false + $streamReader = New-Object System.IO.StreamReader -Arg $_, $utf8NoBomEncoding + $content = $streamReader.ReadToEnd() + $streamReader.Close() + + $lines = $content -split "`n" + $title = "" + $uri = "" + $isArchived = $false + + $titleLine = $lines | Where-Object { $_.StartsWith('title:') } + $title = $titleLine.Substring(6).Trim() + + $uriLine = $lines | Where-Object { $_.Trim().StartsWith('uri:') } + $uri = $uriLine.Substring(4).Trim() + + $archivedReasonLine = $lines | Where-Object { $_.Replace(' ', '').StartsWith('archivedreason:') } + if ($archivedReasonLine) { + $archivedReason = $archivedReasonLine.Trim().Substring(15).Trim() + $isArchived = $archivedReason -ne 'null' -and $archivedReason -ne '' + } + $filesProcessed.Add($_, 0) $historyFileArray += @{ file = $($_) + title = $title + uri = $uri + isArchived = $isArchived lastUpdated = $lastUpdated lastUpdatedBy = $lastUpdatedBy lastUpdatedByEmail = $lastUpdatedByEmail From e3dd9cda74772662818f3f70d04ace577bad6550 Mon Sep 17 00:00:00 2001 From: Baba Kamyljanov Date: Thu, 5 Sep 2024 13:01:07 +0800 Subject: [PATCH 02/11] fix path --- build-scripts/update-rule-history.ps1 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build-scripts/update-rule-history.ps1 b/build-scripts/update-rule-history.ps1 index 0b0ce2dd7..1e904eefe 100644 --- a/build-scripts/update-rule-history.ps1 +++ b/build-scripts/update-rule-history.ps1 @@ -38,6 +38,8 @@ $historyChangeEntry = $listOfCommits -join "" $historyArray = $historyChangeEntry -split "" $commitSyncHash = ""; +$rulesContentFolder = ./SSW.Rules.Content/rules + $historyArray | Foreach-Object { $historyEntry = $_ -split "" $userDetails = $historyEntry[0] -split "" @@ -56,12 +58,13 @@ $historyArray | Foreach-Object { $fileArray | Where-Object {$_ -Match "^*.md" } | Foreach-Object { if(!$filesProcessed.ContainsKey($_)) { + $fullPath = Join-Path $rulesContentFolder $_ $createdRecord = git log --diff-filter=A --reverse --pretty="%ad%aN%ae" --date=iso-strict -- $_ $createdDetails = $createdRecord -split "" # Read and parse Markdown file to set title, uri, and archived status $utf8NoBomEncoding = New-Object System.Text.UTF8Encoding $false - $streamReader = New-Object System.IO.StreamReader -Arg $_, $utf8NoBomEncoding + $streamReader = New-Object System.IO.StreamReader -Arg $fullPath, $utf8NoBomEncoding $content = $streamReader.ReadToEnd() $streamReader.Close() From c667ab2720059f4ede8cd9567ccd085b32e50f46 Mon Sep 17 00:00:00 2001 From: Baba Kamyljanov Date: Thu, 5 Sep 2024 13:04:47 +0800 Subject: [PATCH 03/11] add double quotation mark --- build-scripts/update-rule-history.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-scripts/update-rule-history.ps1 b/build-scripts/update-rule-history.ps1 index 1e904eefe..74beef40a 100644 --- a/build-scripts/update-rule-history.ps1 +++ b/build-scripts/update-rule-history.ps1 @@ -38,7 +38,7 @@ $historyChangeEntry = $listOfCommits -join "" $historyArray = $historyChangeEntry -split "" $commitSyncHash = ""; -$rulesContentFolder = ./SSW.Rules.Content/rules +$rulesContentFolder = "./SSW.Rules.Content/rules" $historyArray | Foreach-Object { $historyEntry = $_ -split "" From 1d6bb1c5e62400c3a4bdc7b7d7032ac6c0f1c076 Mon Sep 17 00:00:00 2001 From: Baba Kamyljanov Date: Thu, 5 Sep 2024 13:12:04 +0800 Subject: [PATCH 04/11] fix path, remove 'rules' part --- build-scripts/update-rule-history.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-scripts/update-rule-history.ps1 b/build-scripts/update-rule-history.ps1 index 74beef40a..78e4dc6c8 100644 --- a/build-scripts/update-rule-history.ps1 +++ b/build-scripts/update-rule-history.ps1 @@ -38,7 +38,7 @@ $historyChangeEntry = $listOfCommits -join "" $historyArray = $historyChangeEntry -split "" $commitSyncHash = ""; -$rulesContentFolder = "./SSW.Rules.Content/rules" +$rulesContentFolder = "./SSW.Rules.Content/" $historyArray | Foreach-Object { $historyEntry = $_ -split "" From a89c81de30ac71d5c5078f56b10504be07dac1d5 Mon Sep 17 00:00:00 2001 From: Baba Kamyljanov Date: Fri, 6 Sep 2024 10:59:49 +0800 Subject: [PATCH 05/11] added try-catch --- build-scripts/update-rule-history.ps1 | 92 ++++++++++++++------------- 1 file changed, 49 insertions(+), 43 deletions(-) diff --git a/build-scripts/update-rule-history.ps1 b/build-scripts/update-rule-history.ps1 index 78e4dc6c8..cf7134411 100644 --- a/build-scripts/update-rule-history.ps1 +++ b/build-scripts/update-rule-history.ps1 @@ -58,48 +58,54 @@ $historyArray | Foreach-Object { $fileArray | Where-Object {$_ -Match "^*.md" } | Foreach-Object { if(!$filesProcessed.ContainsKey($_)) { - $fullPath = Join-Path $rulesContentFolder $_ - $createdRecord = git log --diff-filter=A --reverse --pretty="%ad%aN%ae" --date=iso-strict -- $_ - $createdDetails = $createdRecord -split "" - - # Read and parse Markdown file to set title, uri, and archived status - $utf8NoBomEncoding = New-Object System.Text.UTF8Encoding $false - $streamReader = New-Object System.IO.StreamReader -Arg $fullPath, $utf8NoBomEncoding - $content = $streamReader.ReadToEnd() - $streamReader.Close() - - $lines = $content -split "`n" - $title = "" - $uri = "" - $isArchived = $false - - $titleLine = $lines | Where-Object { $_.StartsWith('title:') } - $title = $titleLine.Substring(6).Trim() - - $uriLine = $lines | Where-Object { $_.Trim().StartsWith('uri:') } - $uri = $uriLine.Substring(4).Trim() - - $archivedReasonLine = $lines | Where-Object { $_.Replace(' ', '').StartsWith('archivedreason:') } - if ($archivedReasonLine) { - $archivedReason = $archivedReasonLine.Trim().Substring(15).Trim() - $isArchived = $archivedReason -ne 'null' -and $archivedReason -ne '' + try { + $fullPath = Join-Path $rulesContentFolder $_ + $createdRecord = git log --diff-filter=A --reverse --pretty="%ad%aN%ae" --date=iso-strict -- $_ + $createdDetails = $createdRecord -split "" + + # Read and parse Markdown file to set title, uri, and archived status + $utf8NoBomEncoding = New-Object System.Text.UTF8Encoding $false + $streamReader = New-Object System.IO.StreamReader -Arg $fullPath, $utf8NoBomEncoding + $content = $streamReader.ReadToEnd() + $streamReader.Close() + + $lines = $content -split "`n" + $title = "" + $uri = "" + $isArchived = $false + + $titleLine = $lines | Where-Object { $_.StartsWith('title:') } + $title = $titleLine.Substring(6).Trim() + + $uriLine = $lines | Where-Object { $_.Trim().StartsWith('uri:') } + $uri = $uriLine.Substring(4).Trim() + + $archivedReasonLine = $lines | Where-Object { $_.Replace(' ', '').StartsWith('archivedreason:') } + if ($archivedReasonLine) { + $archivedReason = $archivedReasonLine.Trim().Substring(15).Trim() + $isArchived = $archivedReason -ne 'null' -and $archivedReason -ne '' + } + + $filesProcessed.Add($_, 0) + + $historyFileArray += @{ + file = $($_) + title = $title + uri = $uri + isArchived = $isArchived + lastUpdated = $lastUpdated + lastUpdatedBy = $lastUpdatedBy + lastUpdatedByEmail = $lastUpdatedByEmail + created = $createdDetails[0] ?? $lastUpdated + createdBy = $createdDetails[1] ?? $lastUpdatedBy + createdByEmail = $createdDetails[2] ?? $lastUpdatedByEmail + } + + echo $_ } - - $filesProcessed.Add($_, 0) - $historyFileArray += @{ - file = $($_) - title = $title - uri = $uri - isArchived = $isArchived - lastUpdated = $lastUpdated - lastUpdatedBy = $lastUpdatedBy - lastUpdatedByEmail = $lastUpdatedByEmail - created = $createdDetails[0] ?? $lastUpdated - createdBy = $createdDetails[1] ?? $lastUpdatedBy - createdByEmail = $createdDetails[2] ?? $lastUpdatedByEmail + catch { + continue } - - echo $_ } } } @@ -122,11 +128,11 @@ if(![string]::IsNullOrWhiteSpace($commitSyncHash)) $Body = @{ commitHash = $commitSyncHash } - $Result = Invoke-WebRequest -Uri $Uri -Method Post -Body $Body -Headers $Headers + $Response = Invoke-WebRequest -Uri $Uri -Method Post -Body $Body -Headers $Headers } if ($ShouldGenerateHistory) { - echo $historyFileContents + Write-Output $historyFileContents } -echo $commitSyncHash +Write-Output $commitSyncHash From 88bae95f89b9d5810d64cf975ad756a706364b1c Mon Sep 17 00:00:00 2001 From: Baba Kamyljanov Date: Fri, 6 Sep 2024 11:52:28 +0800 Subject: [PATCH 06/11] refactored catch block --- build-scripts/update-rule-history.ps1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build-scripts/update-rule-history.ps1 b/build-scripts/update-rule-history.ps1 index cf7134411..2ecffe403 100644 --- a/build-scripts/update-rule-history.ps1 +++ b/build-scripts/update-rule-history.ps1 @@ -104,7 +104,7 @@ $historyArray | Foreach-Object { echo $_ } catch { - continue + Write-Output "Error processing file $_" } } } @@ -114,6 +114,7 @@ $historyArray | Foreach-Object { if ($ShouldGenerateHistory) { #Step 3: UpdateRuleHistory - Send History Patch to AzureFunction + Write-Host "" $historyFileContents = ConvertTo-Json $historyFileArray $Uri = $AzFunctionBaseUrl + '/api/UpdateRuleHistory' $Headers = @{'x-functions-key' = $UpdateRuleHistoryKey} From c5a10a91b1381b2466a0aadd4b61b529415fe056 Mon Sep 17 00:00:00 2001 From: Baba Kamyljanov Date: Fri, 6 Sep 2024 11:56:03 +0800 Subject: [PATCH 07/11] clean-up --- build-scripts/update-rule-history.ps1 | 1 - 1 file changed, 1 deletion(-) diff --git a/build-scripts/update-rule-history.ps1 b/build-scripts/update-rule-history.ps1 index 2ecffe403..795cd38a4 100644 --- a/build-scripts/update-rule-history.ps1 +++ b/build-scripts/update-rule-history.ps1 @@ -114,7 +114,6 @@ $historyArray | Foreach-Object { if ($ShouldGenerateHistory) { #Step 3: UpdateRuleHistory - Send History Patch to AzureFunction - Write-Host "" $historyFileContents = ConvertTo-Json $historyFileArray $Uri = $AzFunctionBaseUrl + '/api/UpdateRuleHistory' $Headers = @{'x-functions-key' = $UpdateRuleHistoryKey} From d6a337af5c14e97a95c3a9af65ebeb3a05de85da Mon Sep 17 00:00:00 2001 From: Baba Kamyljanov Date: Fri, 6 Sep 2024 18:22:39 +0800 Subject: [PATCH 08/11] added test workflows to update history --- .../test-update-history-template.yml | 65 +++++++++++++++++++ .github/workflows/test-update-history.yml | 23 +++++++ 2 files changed, 88 insertions(+) create mode 100644 .github/workflows/test-update-history-template.yml create mode 100644 .github/workflows/test-update-history.yml diff --git a/.github/workflows/test-update-history-template.yml b/.github/workflows/test-update-history-template.yml new file mode 100644 index 000000000..c32f77bc4 --- /dev/null +++ b/.github/workflows/test-update-history-template.yml @@ -0,0 +1,65 @@ +name: Template - Build site + +on: + workflow_call: + inputs: + branch_name: + required: true + type: string + environment: + type: string + required: true + description: 'The environment to build for' + should_generate_commit_data: + type: boolean + description: 'Generate commit data for rules history. You might want to skip this on large rules changes.' + required: false + default: true + +jobs: + get-content-commit-data: + name: Generate commit data + runs-on: ubuntu-latest + environment: ${{ inputs.environment }} + steps: + + - uses: actions/checkout@v4 + name: Checkout SSW.Rules + with: + path: SSW.Rules + ref: ${{ inputs.branch_name }} + sparse-checkout: | + build-scripts + fetch-depth: 1 + + - uses: actions/checkout@v4 + name: Checkout SSW.Rules.Content + with: + repository: SSWConsulting/SSW.Rules.Content + path: SSW.Rules.Content + ref: main + fetch-depth: 0 + + - name: Record the latest SHA for the content repo + working-directory: SSW.Rules.Content + run: | + echo '### Content repo info! 🚀' >> $GITHUB_STEP_SUMMARY + echo "Latest SHA: [$(git rev-parse HEAD)](https://github.com/SSWConsulting/SSW.Rules.Content/commit/$(git rev-parse HEAD))" >> $GITHUB_STEP_SUMMARY + + + - name: Get Rule History Commits + shell: pwsh + run: | + ./SSW.Rules/build-scripts/update-rule-history.ps1 ` + -AzFunctionBaseUrl ${{ vars.AzFunctionBaseUrl }} ` + -GetHistorySyncCommitHashKey ${{ secrets.GETHISTORYSYNCCOMMITHASHFUNCTIONKEY }} ` + -UpdateRuleHistoryKey ${{ secrets.UPDATERULEHISTORYFUNCTIONKEY }} ` + -UpdateHistorySyncCommitHashKey ${{ secrets.UPDATEHISTORYSYNCCOMMITHASHFUNCTIONKEY }} ` + -ShouldGenerateHistory ${{ inputs.should_generate_commit_data }} + + - name: Archive artifacts + uses: actions/upload-artifact@v4 + with: + name: commit-data + path: static + retention-days: 1 diff --git a/.github/workflows/test-update-history.yml b/.github/workflows/test-update-history.yml new file mode 100644 index 000000000..71471d037 --- /dev/null +++ b/.github/workflows/test-update-history.yml @@ -0,0 +1,23 @@ +name: Staging - build and deploy + +on: + push: + branches: + - 1438-fix-widget-data + workflow_dispatch: + inputs: + should_generate_commit_data: + type: boolean + description: 'Generate commit data' + default: true + +jobs: + + build: + name: Build + uses: ./.github/workflows/template-build.yml + with: + branch_name: ${{ github.ref }} + environment: staging + should_generate_commit_data: ${{ github.event.inputs.should_generate_commit_data == 'true' }} + secrets: inherit \ No newline at end of file From fb8d65e68be6a1e91d98c102f59f3b72376ed1ca Mon Sep 17 00:00:00 2001 From: Baba Kamyljanov Date: Fri, 6 Sep 2024 18:27:16 +0800 Subject: [PATCH 09/11] fix path --- .github/workflows/test-update-history.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-update-history.yml b/.github/workflows/test-update-history.yml index 71471d037..af35646eb 100644 --- a/.github/workflows/test-update-history.yml +++ b/.github/workflows/test-update-history.yml @@ -15,7 +15,7 @@ jobs: build: name: Build - uses: ./.github/workflows/template-build.yml + uses: ./.github/workflows/test-update-history-template.yml with: branch_name: ${{ github.ref }} environment: staging From 76ab96e11b74e46c63f354daea5418022eeb9c72 Mon Sep 17 00:00:00 2001 From: Baba Kamyljanov Date: Fri, 6 Sep 2024 18:40:39 +0800 Subject: [PATCH 10/11] clean up --- .../test-update-history-template.yml | 65 ------------------- .github/workflows/test-update-history.yml | 23 ------- 2 files changed, 88 deletions(-) delete mode 100644 .github/workflows/test-update-history-template.yml delete mode 100644 .github/workflows/test-update-history.yml diff --git a/.github/workflows/test-update-history-template.yml b/.github/workflows/test-update-history-template.yml deleted file mode 100644 index c32f77bc4..000000000 --- a/.github/workflows/test-update-history-template.yml +++ /dev/null @@ -1,65 +0,0 @@ -name: Template - Build site - -on: - workflow_call: - inputs: - branch_name: - required: true - type: string - environment: - type: string - required: true - description: 'The environment to build for' - should_generate_commit_data: - type: boolean - description: 'Generate commit data for rules history. You might want to skip this on large rules changes.' - required: false - default: true - -jobs: - get-content-commit-data: - name: Generate commit data - runs-on: ubuntu-latest - environment: ${{ inputs.environment }} - steps: - - - uses: actions/checkout@v4 - name: Checkout SSW.Rules - with: - path: SSW.Rules - ref: ${{ inputs.branch_name }} - sparse-checkout: | - build-scripts - fetch-depth: 1 - - - uses: actions/checkout@v4 - name: Checkout SSW.Rules.Content - with: - repository: SSWConsulting/SSW.Rules.Content - path: SSW.Rules.Content - ref: main - fetch-depth: 0 - - - name: Record the latest SHA for the content repo - working-directory: SSW.Rules.Content - run: | - echo '### Content repo info! 🚀' >> $GITHUB_STEP_SUMMARY - echo "Latest SHA: [$(git rev-parse HEAD)](https://github.com/SSWConsulting/SSW.Rules.Content/commit/$(git rev-parse HEAD))" >> $GITHUB_STEP_SUMMARY - - - - name: Get Rule History Commits - shell: pwsh - run: | - ./SSW.Rules/build-scripts/update-rule-history.ps1 ` - -AzFunctionBaseUrl ${{ vars.AzFunctionBaseUrl }} ` - -GetHistorySyncCommitHashKey ${{ secrets.GETHISTORYSYNCCOMMITHASHFUNCTIONKEY }} ` - -UpdateRuleHistoryKey ${{ secrets.UPDATERULEHISTORYFUNCTIONKEY }} ` - -UpdateHistorySyncCommitHashKey ${{ secrets.UPDATEHISTORYSYNCCOMMITHASHFUNCTIONKEY }} ` - -ShouldGenerateHistory ${{ inputs.should_generate_commit_data }} - - - name: Archive artifacts - uses: actions/upload-artifact@v4 - with: - name: commit-data - path: static - retention-days: 1 diff --git a/.github/workflows/test-update-history.yml b/.github/workflows/test-update-history.yml deleted file mode 100644 index af35646eb..000000000 --- a/.github/workflows/test-update-history.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: Staging - build and deploy - -on: - push: - branches: - - 1438-fix-widget-data - workflow_dispatch: - inputs: - should_generate_commit_data: - type: boolean - description: 'Generate commit data' - default: true - -jobs: - - build: - name: Build - uses: ./.github/workflows/test-update-history-template.yml - with: - branch_name: ${{ github.ref }} - environment: staging - should_generate_commit_data: ${{ github.event.inputs.should_generate_commit_data == 'true' }} - secrets: inherit \ No newline at end of file From 144ebaa1dd862222059779c30862f49e00d026f3 Mon Sep 17 00:00:00 2001 From: Baba Kamyljanov Date: Tue, 10 Sep 2024 10:39:37 +0800 Subject: [PATCH 11/11] refactor --- build-scripts/update-rule-history.ps1 | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/build-scripts/update-rule-history.ps1 b/build-scripts/update-rule-history.ps1 index 795cd38a4..0cb52c870 100644 --- a/build-scripts/update-rule-history.ps1 +++ b/build-scripts/update-rule-history.ps1 @@ -10,9 +10,9 @@ param ( ) if ($ShouldGenerateHistory -eq $false) { - echo "Skipping history generation" + Write-Output "Skipping history generation" } else { - echo "Generating history" + Write-Output "Generating history" } @@ -101,7 +101,7 @@ $historyArray | Foreach-Object { createdByEmail = $createdDetails[2] ?? $lastUpdatedByEmail } - echo $_ + Write-Output $_ } catch { Write-Output "Error processing file $_" @@ -128,11 +128,11 @@ if(![string]::IsNullOrWhiteSpace($commitSyncHash)) $Body = @{ commitHash = $commitSyncHash } - $Response = Invoke-WebRequest -Uri $Uri -Method Post -Body $Body -Headers $Headers + $Result = Invoke-WebRequest -Uri $Uri -Method Post -Body $Body -Headers $Headers } if ($ShouldGenerateHistory) { - Write-Output $historyFileContents + echo $historyFileContents } -Write-Output $commitSyncHash +echo $commitSyncHash