From 894e13378c8ff809b489cc16b84747d15ad395b8 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Mon, 4 Nov 2024 17:37:22 -0800 Subject: [PATCH 01/10] convert python to using common matrix generation --- eng/pipelines/templates/jobs/ci.yml | 22 +++----- .../steps/build-package-artifacts.yml | 54 ++----------------- 2 files changed, 11 insertions(+), 65 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index ef7191fff84d..7a7e6bccd2d7 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -244,20 +244,14 @@ jobs: MatrixConfigs: ${{ parameters.MatrixConfigs }} MatrixFilters: ${{ parameters.MatrixFilters }} MatrixReplace: ${{ parameters.MatrixReplace }} - PreGenerationSteps: - - - template: /eng/pipelines/templates/steps/download-package-artifacts.yml - - - task: Powershell@2 - inputs: - pwsh: true - filePath: eng/scripts/distribute-packages-to-matrix.ps1 - arguments: >- - -PackageInfoFolder "$(Build.ArtifactStagingDirectory)/PackageInfo" - -PlatformMatrix "${{ parameters.MatrixConfigs[0].Path }}" - displayName: 'Distribute Packages to Matrix' - condition: and(eq(variables['Build.Reason'], 'PullRequest'), eq('${{ parameters.ServiceDirectory }}','auto')) - + ${{ if eq(parameters.ServiceDirectory, 'auto') }}: + SparseCheckoutPaths: [ "**/setup.py", "**/ci*.yml"] + ${{ if eq(parameters.ServiceDirectory, 'auto') }}: + EnablePRGeneration: true + PreGenerationSteps: + - template: /eng/common/pipelines/templates/steps/save-package-properties.yml + parameters: + ServiceDirectory: ${{parameters.ServiceDirectory}} CloudConfig: Cloud: Public AdditionalParameters: diff --git a/eng/pipelines/templates/steps/build-package-artifacts.yml b/eng/pipelines/templates/steps/build-package-artifacts.yml index 784a31ce0855..ddd4e4ab2e79 100644 --- a/eng/pipelines/templates/steps/build-package-artifacts.yml +++ b/eng/pipelines/templates/steps/build-package-artifacts.yml @@ -59,57 +59,9 @@ steps: parameters: ServiceDirectory: ${{ parameters.ServiceDirectory }} - # now we need to call Save-Package-Properties so that we can filter on it - - ${{ if and(eq(variables['Build.Reason'], 'PullRequest'), eq(parameters.ServiceDirectory, 'auto')) }}: - - pwsh: | - mkdir -p $(Build.ArtifactStagingDirectory)/diff - displayName: Create PR Diff Folder - - - pwsh: | - $location = Join-Path "$(Build.ArtifactStagingDirectory)" "diff" - - Write-Host "./eng/common/scripts/Generate-PR-Diff.ps1 -TargetPath `"$(Build.SourcesDirectory)`" -ArtifactPath `"$location`"" - ./eng/common/scripts/Generate-PR-Diff.ps1 -TargetPath "$(Build.SourcesDirectory)" -ArtifactPath "$location" - displayName: Generate PR Diff - - - pwsh: | - Write-Host "We freshly generated the PR diff, and this is what it sees!" - Get-ChildItem -R -Force $(Build.ArtifactStagingDirectory)/diff | % { $_.FullName } - cat $(Build.ArtifactStagingDirectory)/diff/diff.json - displayName: Dump PR Diff - - - task: Powershell@2 - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/scripts/Save-Package-Properties.ps1 - arguments: > - -PrDiff $(Build.ArtifactStagingDirectory)/diff/diff.json - -OutDirectory $(Build.ArtifactStagingDirectory)/PackageInfo - pwsh: true - workingDirectory: $(Pipeline.Workspace) - displayName: Save package properties filtered for PR - - - ${{ else }}: - - task: Powershell@2 - condition: and(succeeded(), eq(variables['SetDevVersion'],'true')) - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/scripts/Save-Package-Properties.ps1 - arguments: > - -ServiceDirectory ${{parameters.ServiceDirectory}} - -OutDirectory $(Build.ArtifactStagingDirectory)/PackageInfo - -AddDevVersion - pwsh: true - workingDirectory: $(Pipeline.Workspace) - displayName: Save package properties with dev version - - task: Powershell@2 - condition: and(succeeded(), ne(variables['SetDevVersion'],'true')) - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/scripts/Save-Package-Properties.ps1 - arguments: > - -ServiceDirectory ${{parameters.ServiceDirectory}} - -OutDirectory $(Build.ArtifactStagingDirectory)/PackageInfo - pwsh: true - workingDirectory: $(Pipeline.Workspace) - displayName: Save package properties for service + - template: /eng/common/save-package-properties.yml + parameters: + ServiceDirectory: ${{ parameters.ServiceDirectory }} - template: /eng/pipelines/templates/steps/resolve-package-targeting.yml parameters: From 3b6481e14eee0891f4ee3d67175425209a67ed6d Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Mon, 4 Nov 2024 17:41:30 -0800 Subject: [PATCH 02/10] correct yml reference --- eng/pipelines/templates/steps/build-package-artifacts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/steps/build-package-artifacts.yml b/eng/pipelines/templates/steps/build-package-artifacts.yml index ddd4e4ab2e79..21f1794f7477 100644 --- a/eng/pipelines/templates/steps/build-package-artifacts.yml +++ b/eng/pipelines/templates/steps/build-package-artifacts.yml @@ -59,7 +59,7 @@ steps: parameters: ServiceDirectory: ${{ parameters.ServiceDirectory }} - - template: /eng/common/save-package-properties.yml + - template: /eng/common/pipelines/templates/steps/save-package-properties.yml parameters: ServiceDirectory: ${{ parameters.ServiceDirectory }} From 2f87ea9541420f3cb6783b5acc5d9c6f73cf4af3 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Mon, 4 Nov 2024 18:05:50 -0800 Subject: [PATCH 03/10] update checkout logic --- eng/pipelines/templates/jobs/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 7a7e6bccd2d7..239574bc7373 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -245,7 +245,7 @@ jobs: MatrixFilters: ${{ parameters.MatrixFilters }} MatrixReplace: ${{ parameters.MatrixReplace }} ${{ if eq(parameters.ServiceDirectory, 'auto') }}: - SparseCheckoutPaths: [ "**/setup.py", "**/ci*.yml"] + SparseCheckoutPaths: [ "**/setup.py", "**/pyproject.toml", "tools/", "**/ci*.yml"] ${{ if eq(parameters.ServiceDirectory, 'auto') }}: EnablePRGeneration: true PreGenerationSteps: From 306e96c4f297d0803f8f55f6b40af6d3a0d2e331 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 5 Nov 2024 13:41:40 -0800 Subject: [PATCH 04/10] update pregeneration steps so we can see what is going wrong in the get-package-properties call --- eng/pipelines/templates/jobs/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 239574bc7373..45145e48b96d 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -249,6 +249,9 @@ jobs: ${{ if eq(parameters.ServiceDirectory, 'auto') }}: EnablePRGeneration: true PreGenerationSteps: + - pwsh: | + python -m pip install "./tools/azure-sdk-tools[build]" + displayName: Install Azure SDK Tools To Detect Install Failure - template: /eng/common/pipelines/templates/steps/save-package-properties.yml parameters: ServiceDirectory: ${{parameters.ServiceDirectory}} From b0b155509fb7ecfa5ed8f6060d0d2174183dd3a3 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 12 Nov 2024 16:30:04 -0800 Subject: [PATCH 05/10] ensure we can see what is breaking --- eng/pipelines/templates/jobs/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 45145e48b96d..ee0013b439ee 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -252,6 +252,10 @@ jobs: - pwsh: | python -m pip install "./tools/azure-sdk-tools[build]" displayName: Install Azure SDK Tools To Detect Install Failure + - pwsh: | + git --version + Get-ChildItem -Recurse -Force | % { Write-Host $_.FullName } + displayName: Dump visible package files - template: /eng/common/pipelines/templates/steps/save-package-properties.yml parameters: ServiceDirectory: ${{parameters.ServiceDirectory}} From 376174d6844fb5046428e5875f7cf321c6ed0b59 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 12 Nov 2024 16:45:27 -0800 Subject: [PATCH 06/10] adding some additional activity, what is going wrong? --- eng/pipelines/templates/jobs/ci.yml | 6 +++--- eng/scripts/Language-Settings.ps1 | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index ee0013b439ee..0a5be26072a8 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -249,9 +249,9 @@ jobs: ${{ if eq(parameters.ServiceDirectory, 'auto') }}: EnablePRGeneration: true PreGenerationSteps: - - pwsh: | - python -m pip install "./tools/azure-sdk-tools[build]" - displayName: Install Azure SDK Tools To Detect Install Failure + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.12' - pwsh: | git --version Get-ChildItem -Recurse -Force | % { Write-Host $_.FullName } diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index b65b4b43ba53..da51888306e1 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -137,6 +137,7 @@ function Get-AllPackageInfoFromRepo ($serviceDirectory) Push-Location $RepoRoot $null = python -m pip install "./tools/azure-sdk-tools[build]" -q -I $allPkgPropLines = python (Join-path eng scripts get_package_properties.py) -s $searchPath + Write-Host $allPkgPropLines } catch { From 5c54bed01937c3187d4e57990310716bff46130b Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 13 Nov 2024 13:52:55 -0800 Subject: [PATCH 07/10] pulling readme, changelog, and versioning files so we can successfully parse the setup.py of each package --- eng/pipelines/templates/jobs/ci.yml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 0a5be26072a8..a48e3681e94e 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -245,17 +245,21 @@ jobs: MatrixFilters: ${{ parameters.MatrixFilters }} MatrixReplace: ${{ parameters.MatrixReplace }} ${{ if eq(parameters.ServiceDirectory, 'auto') }}: - SparseCheckoutPaths: [ "**/setup.py", "**/pyproject.toml", "tools/", "**/ci*.yml"] - ${{ if eq(parameters.ServiceDirectory, 'auto') }}: + SparseCheckoutPaths: + - "**/setup.py" + - "**/pyproject.toml" + - "tools/" + - "**/ci*.yml" + - "**/_version.py" + - "**/version.py" + - "**/README.md" + - "**/CHANGELOG.md" EnablePRGeneration: true + PRMatrixSetting: "TargetingString" PreGenerationSteps: - task: UsePythonVersion@0 inputs: versionSpec: '3.12' - - pwsh: | - git --version - Get-ChildItem -Recurse -Force | % { Write-Host $_.FullName } - displayName: Dump visible package files - template: /eng/common/pipelines/templates/steps/save-package-properties.yml parameters: ServiceDirectory: ${{parameters.ServiceDirectory}} From d675e01788e9bc4a2a3a83a461d96de13b1eeec6 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Fri, 15 Nov 2024 13:16:47 -0800 Subject: [PATCH 08/10] ensure that an identity change will unfold properly --- sdk/identity/azure-identity/LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/identity/azure-identity/LICENSE b/sdk/identity/azure-identity/LICENSE index 63447fd8bbbf..b2f52a2bad4e 100644 --- a/sdk/identity/azure-identity/LICENSE +++ b/sdk/identity/azure-identity/LICENSE @@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file +SOFTWARE. From cb436665ed57e9264a9218cdb970720e5d122854 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Fri, 15 Nov 2024 13:37:21 -0800 Subject: [PATCH 09/10] some further cleanup + update of the sparse checkout to have the platform-matrix files available for parsing --- eng/pipelines/templates/jobs/ci.yml | 1 + sdk/attestation/ci.yml | 6 ------ sdk/evaluation/ci.yml | 7 ++----- sdk/keyvault/ci.yml | 6 ------ 4 files changed, 3 insertions(+), 17 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index a48e3681e94e..7e587a2e8762 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -254,6 +254,7 @@ jobs: - "**/version.py" - "**/README.md" - "**/CHANGELOG.md" + - "**/*matrix*.json" EnablePRGeneration: true PRMatrixSetting: "TargetingString" PreGenerationSteps: diff --git a/sdk/attestation/ci.yml b/sdk/attestation/ci.yml index 9eb456cc2268..118897a02e1e 100644 --- a/sdk/attestation/ci.yml +++ b/sdk/attestation/ci.yml @@ -28,12 +28,6 @@ extends: parameters: ServiceDirectory: attestation TestProxy: true - # Override the base matrix due to https://github.com/Azure/azure-sdk-for-python/issues/17837 - MatrixConfigs: - - Name: attestation_ci_matrix - Path: eng/pipelines/templates/stages/platform-matrix.json - Selection: sparse - GenerateVMJobs: true Artifacts: - name: azure-mgmt-attestation safeName: azuremgmtattestation diff --git a/sdk/evaluation/ci.yml b/sdk/evaluation/ci.yml index cfd1a9f43ae7..628963561d6d 100644 --- a/sdk/evaluation/ci.yml +++ b/sdk/evaluation/ci.yml @@ -30,11 +30,8 @@ extends: TestProxy: true # This custom matrix config should be dropped once: # * Once azure-ai-ml supports 3.13 (currently crashes due to type annotation) - MatrixConfigs: - - Name: evaluation_ci_matrix - Path: eng/pipelines/templates/stages/platform-matrix-no-313.json - Selection: sparse - GenerateVMJobs: true + MatrixFilters: + - PythonVersion=^(?!3\.13) Artifacts: - name: azure-ai-evaluation safeName: azureaievaluation diff --git a/sdk/keyvault/ci.yml b/sdk/keyvault/ci.yml index 7417cb7337ee..c69612d67ec7 100644 --- a/sdk/keyvault/ci.yml +++ b/sdk/keyvault/ci.yml @@ -31,12 +31,6 @@ extends: parameters: ServiceDirectory: keyvault TestProxy: true - # Override the base matrix due to https://github.com/Azure/azure-sdk-for-python/issues/17837 - MatrixConfigs: - - Name: keyvault_ci_matrix - Path: eng/pipelines/templates/stages/platform-matrix.json - Selection: sparse - GenerateVMJobs: true Artifacts: - name: azure-keyvault-administration safename: azurekeyvaultadministration From 758500afd74c3a7ee2204385a5d8651543a03bee Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Fri, 15 Nov 2024 14:25:03 -0800 Subject: [PATCH 10/10] remove extraneous line from language settings. --- eng/scripts/Language-Settings.ps1 | 1 - 1 file changed, 1 deletion(-) diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index da51888306e1..b65b4b43ba53 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -137,7 +137,6 @@ function Get-AllPackageInfoFromRepo ($serviceDirectory) Push-Location $RepoRoot $null = python -m pip install "./tools/azure-sdk-tools[build]" -q -I $allPkgPropLines = python (Join-path eng scripts get_package_properties.py) -s $searchPath - Write-Host $allPkgPropLines } catch {