Skip to content

Commit 8e207b7

Browse files
azure-sdkscbedd
azure-sdk
and
scbedd
authored
Sync eng/common directory with azure-sdk-tools for PR 9281 (#38213)
* package-properties are now populated with matrix configurations from their ci.yml if present * create new code path for generate-job-matrix.yml which combines Create-JobMatrix and the "distribute-packages-to-matrix" action to generate dynamic matrices for PRs --------- Co-authored-by: Scott Beddall <[email protected]>
1 parent f306890 commit 8e207b7

File tree

4 files changed

+340
-113
lines changed

4 files changed

+340
-113
lines changed

‎eng/common/pipelines/templates/jobs/generate-job-matrix.yml

+86-50
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,12 @@ parameters:
4242
- name: PreGenerationSteps
4343
type: stepList
4444
default: []
45+
- name: EnablePRGeneration
46+
type: boolean
47+
default: false
48+
- name: PRMatrixSetting
49+
type: string
50+
default: 'ArtifactPackageNames'
4551
# Mappings to OS name required at template compile time by 1es pipeline templates
4652
- name: Pools
4753
type: object
@@ -84,57 +90,87 @@ jobs:
8490

8591
- ${{ parameters.PreGenerationSteps }}
8692

87-
- ${{ each config in parameters.MatrixConfigs }}:
93+
- ${{ if eq(parameters.EnablePRGeneration, false) }}:
94+
- ${{ each config in parameters.MatrixConfigs }}:
95+
- ${{ each pool in parameters.Pools }}:
96+
- ${{ if eq(config.GenerateVMJobs, 'true') }}:
97+
- task: Powershell@2
98+
inputs:
99+
pwsh: true
100+
filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1
101+
arguments: >
102+
-ConfigPath ${{ config.Path }}
103+
-Selection ${{ config.Selection }}
104+
-DisplayNameFilter '$(displayNameFilter)'
105+
-Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}'
106+
-Replace '${{ join(''',''', parameters.MatrixReplace) }}'
107+
-NonSparseParameters '${{ join(''',''', config.NonSparseParameters) }}'
108+
displayName: Create ${{ pool.name }} Matrix ${{ config.Name }}
109+
name: vm_job_matrix_${{ config.Name }}_${{ pool.name }}
110+
- ${{ if eq(config.GenerateContainerJobs, 'true') }}:
111+
- task: Powershell@2
112+
inputs:
113+
pwsh: true
114+
filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1
115+
arguments: >
116+
-ConfigPath ${{ config.Path }}
117+
-Selection ${{ config.Selection }}
118+
-DisplayNameFilter '$(displayNameFilter)'
119+
-Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}'
120+
-NonSparseParameters '${{ join(''',''', config.NonSparseParameters) }}'
121+
displayName: Create ${{ pool.name }} Container Matrix ${{ config.Name }}
122+
name: container_job_matrix_${{ config.Name }}_${{ pool.name }}
123+
124+
# This else being set also currently assumes that the $(Build.ArtifactStagingDirectory)/PackageInfo folder is populated by PreGenerationSteps.
125+
# Not currently not hardcoded, so not doing the needful and populating this folder before we hit this step will result in generation errors.
126+
- ${{ else }}:
88127
- ${{ each pool in parameters.Pools }}:
89-
- ${{ if eq(config.GenerateVMJobs, 'true') }}:
90-
- task: Powershell@2
91-
inputs:
92-
pwsh: true
93-
filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1
94-
arguments: >
95-
-ConfigPath ${{ config.Path }}
96-
-Selection ${{ config.Selection }}
97-
-DisplayNameFilter '$(displayNameFilter)'
98-
-Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}'
99-
-Replace '${{ join(''',''', parameters.MatrixReplace) }}'
100-
-NonSparseParameters '${{ join(''',''', config.NonSparseParameters) }}'
101-
displayName: Create ${{ pool.name }} Matrix ${{ config.Name }}
102-
name: vm_job_matrix_${{ config.Name }}_${{ pool.name }}
128+
- pwsh: |
129+
# dump the conglomerated CI matrix
130+
'${{ convertToJson(parameters.MatrixConfigs) }}' | Set-Content matrix.json
103131
104-
- ${{ if eq(config.GenerateContainerJobs, 'true') }}:
105-
- task: Powershell@2
106-
inputs:
107-
pwsh: true
108-
filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1
109-
arguments: >
110-
-ConfigPath ${{ config.Path }}
111-
-Selection ${{ config.Selection }}
112-
-DisplayNameFilter '$(displayNameFilter)'
113-
-Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}'
114-
-NonSparseParameters '${{ join(''',''', config.NonSparseParameters) }}'
115-
displayName: Create ${{ pool.name }} Container Matrix ${{ config.Name }}
116-
name: container_job_matrix_${{ config.Name }}_${{ pool.name }}
132+
./eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 `
133+
-PackagePropertiesFolder $(Build.ArtifactStagingDirectory)/PackageInfo `
134+
-PRMatrixFile matrix.json `
135+
-PRMatrixSetting ${{ parameters.PRMatrixSetting }} `
136+
-DisplayNameFilter '$(displayNameFilter)' `
137+
-Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}' `
138+
-Replace '${{ join(''',''', parameters.MatrixReplace) }}'
139+
displayName: Create ${{ pool.name }} PR Matrix
140+
name: vm_job_matrix_pr_${{ pool.name }}
117141
118-
- ${{ each config in parameters.MatrixConfigs }}:
119-
- ${{ each pool in parameters.Pools }}:
120-
- ${{ if eq(config.GenerateVMJobs, 'true') }}:
121-
- template: ${{ parameters.JobTemplatePath }}
122-
parameters:
123-
UsePlatformContainer: false
124-
OSName: ${{ pool.os }}
125-
Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_${{ config.Name }}_${{ pool.name }}.matrix']
126-
DependsOn: ${{ parameters.GenerateJobName }}
127-
CloudConfig: ${{ parameters.CloudConfig }}
128-
${{ each param in parameters.AdditionalParameters }}:
129-
${{ param.key }}: ${{ param.value }}
142+
- ${{ if eq(parameters.EnablePRGeneration, false) }}:
143+
- ${{ each config in parameters.MatrixConfigs }}:
144+
- ${{ each pool in parameters.Pools }}:
145+
- ${{ if eq(config.GenerateVMJobs, 'true') }}:
146+
- template: ${{ parameters.JobTemplatePath }}
147+
parameters:
148+
UsePlatformContainer: false
149+
OSName: ${{ pool.os }}
150+
Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_${{ config.Name }}_${{ pool.name }}.matrix']
151+
DependsOn: ${{ parameters.GenerateJobName }}
152+
CloudConfig: ${{ parameters.CloudConfig }}
153+
${{ each param in parameters.AdditionalParameters }}:
154+
${{ param.key }}: ${{ param.value }}
130155

131-
- ${{ if eq(config.GenerateContainerJobs, 'true') }}:
132-
- template: ${{ parameters.JobTemplatePath }}
133-
parameters:
134-
UsePlatformContainer: true
135-
OSName: ${{ pool.os }}
136-
Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_${{ config.Name }}_${{ pool.name }}.matrix']
137-
DependsOn: ${{ parameters.GenerateJobName }}
138-
CloudConfig: ${{ parameters.CloudConfig }}
139-
${{ each param in parameters.AdditionalParameters }}:
140-
${{ param.key }}: ${{ param.value }}
156+
- ${{ if eq(config.GenerateContainerJobs, 'true') }}:
157+
- template: ${{ parameters.JobTemplatePath }}
158+
parameters:
159+
UsePlatformContainer: true
160+
OSName: ${{ pool.os }}
161+
Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_${{ config.Name }}_${{ pool.name }}.matrix']
162+
DependsOn: ${{ parameters.GenerateJobName }}
163+
CloudConfig: ${{ parameters.CloudConfig }}
164+
${{ each param in parameters.AdditionalParameters }}:
165+
${{ param.key }}: ${{ param.value }}
166+
- ${{ else }}:
167+
- ${{ each pool in parameters.Pools }}:
168+
- template: ${{ parameters.JobTemplatePath }}
169+
parameters:
170+
UsePlatformContainer: false
171+
OSName: ${{ pool.os }}
172+
Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_pr_${{ pool.name }}.matrix']
173+
DependsOn: ${{ parameters.GenerateJobName }}
174+
CloudConfig: ${{ parameters.CloudConfig }}
175+
${{ each param in parameters.AdditionalParameters }}:
176+
${{ param.key }}: ${{ param.value }}

‎eng/common/scripts/Helpers/Package-Helpers.ps1

+78-2
Original file line numberDiff line numberDiff line change
@@ -170,10 +170,86 @@ function GetValueSafelyFrom-Yaml {
170170
$current = $current[$key]
171171
}
172172
else {
173-
Write-Host "The '$key' part of the path $($Keys -join "/") doesn't exist or is null."
174173
return $null
175174
}
176175
}
177176

178177
return [object]$current
179-
}
178+
}
179+
180+
function Get-ObjectKey {
181+
param (
182+
[Parameter(Mandatory = $true)]
183+
[object]$Object
184+
)
185+
186+
if (-not $Object) {
187+
return "unset"
188+
}
189+
190+
if ($Object -is [hashtable] -or $Object -is [System.Collections.Specialized.OrderedDictionary]) {
191+
$sortedEntries = $Object.GetEnumerator() | Sort-Object Name
192+
$hashString = ($sortedEntries | ForEach-Object { "$($_.Key)=$($_.Value)" }) -join ";"
193+
return $hashString.GetHashCode()
194+
}
195+
196+
elseif ($Object -is [PSCustomObject]) {
197+
$sortedProperties = $Object.PSObject.Properties | Sort-Object Name
198+
$propertyString = ($sortedProperties | ForEach-Object { "$($_.Name)=$($_.Value)" }) -join ";"
199+
return $propertyString.GetHashCode()
200+
}
201+
202+
elseif ($Object -is [array]) {
203+
$arrayString = ($Object | ForEach-Object { Get-ObjectKey $_ }) -join ";"
204+
return $arrayString.GetHashCode()
205+
}
206+
207+
else {
208+
return $Object.GetHashCode()
209+
}
210+
}
211+
212+
function Group-ByObjectKey {
213+
param (
214+
[Parameter(Mandatory)]
215+
[array]$Items,
216+
217+
[Parameter(Mandatory)]
218+
[string]$GroupByProperty
219+
)
220+
221+
$groupedDictionary = @{}
222+
223+
foreach ($item in $Items) {
224+
$key = Get-ObjectKey $item."$GroupByProperty"
225+
226+
if (-not $groupedDictionary.ContainsKey($key)) {
227+
$groupedDictionary[$key] = @()
228+
}
229+
230+
# Add the current item to the array for this key
231+
$groupedDictionary[$key] += $item
232+
}
233+
234+
return $groupedDictionary
235+
}
236+
237+
function Split-ArrayIntoBatches {
238+
param (
239+
[Parameter(Mandatory = $true)]
240+
[Object[]]$InputArray,
241+
242+
[Parameter(Mandatory = $true)]
243+
[int]$BatchSize
244+
)
245+
246+
$batches = @()
247+
248+
for ($i = 0; $i -lt $InputArray.Count; $i += $BatchSize) {
249+
$batch = $InputArray[$i..[math]::Min($i + $BatchSize - 1, $InputArray.Count - 1)]
250+
251+
$batches += , $batch
252+
}
253+
254+
return , $batches
255+
}

0 commit comments

Comments
 (0)