Skip to content

Commit 182b5a0

Browse files
azure-sdkckairen
azure-sdk
and
ckairen
authored
Sync eng/common directory with azure-sdk-tools for PR 3790 (#26927)
* Added yaml support for job matrix creation * autogen scenario matrix for stress test * Temporary Working State * update to default sparse * pr comments and some error handling * custom matrixfilename and ordering of generatedValues.yaml * common module import * JobMatrix write host Co-authored-by: Albert Cheng <[email protected]>
1 parent 6bf1b90 commit 182b5a0

7 files changed

+251
-57
lines changed

‎eng/common/scripts/Helpers/PSModule-Helpers.ps1

+3-3
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,9 @@ function Update-PSModulePathForCI()
2222
$modulePaths = $modulePaths.Where({ !$_.StartsWith($hostedAgentModulePath) })
2323

2424
# Add any "az_" paths from the agent which is the lastest set of azure modules
25-
$AzModuleCachPath = (Get-ChildItem "$hostedAgentModulePath/az_*" -Attributes Directory) -join $moduleSeperator
26-
if ($AzModuleCachPath -and $env.PSModulePath -notcontains $AzModuleCachPath) {
27-
$modulePaths += $AzModuleCachPath
25+
$AzModuleCachePath = (Get-ChildItem "$hostedAgentModulePath/az_*" -Attributes Directory) -join $moduleSeperator
26+
if ($AzModuleCachePath -and $env:PSModulePath -notcontains $AzModuleCachePath) {
27+
$modulePaths += $AzModuleCachePath
2828
}
2929

3030
$env:PSModulePath = $modulePaths -join $moduleSeperator

‎eng/common/scripts/job-matrix/Create-JobMatrix.ps1

+5-2
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ if (!(Test-Path $ConfigPath)) {
2323
Write-Error "ConfigPath '$ConfigPath' does not exist."
2424
exit 1
2525
}
26-
$config = GetMatrixConfigFromJson (Get-Content $ConfigPath)
26+
$config = GetMatrixConfigFromFile (Get-Content $ConfigPath -Raw)
2727
# Strip empty string filters in order to be able to use azure pipelines yaml join()
2828
$Filters = $Filters | Where-Object { $_ }
2929

@@ -38,4 +38,7 @@ $Filters = $Filters | Where-Object { $_ }
3838
$serialized = SerializePipelineMatrix $matrix
3939

4040
Write-Output $serialized.pretty
41-
Write-Output "##vso[task.setVariable variable=matrix;isOutput=true]$($serialized.compressed)"
41+
42+
if ($null -ne $env:SYSTEM_TEAMPROJECTID) {
43+
Write-Output "##vso[task.setVariable variable=matrix;isOutput=true]$($serialized.compressed)"
44+
}

‎eng/common/scripts/job-matrix/job-matrix-functions.ps1

+29-5
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ class MatrixParameter {
8484
}
8585
}
8686

87+
. (Join-Path $PSScriptRoot "../Helpers" PSModule-Helpers.ps1)
8788
$IMPORT_KEYWORD = '$IMPORT'
8889

8990
function GenerateMatrix(
@@ -146,7 +147,7 @@ function ProcessNonSparseParameters(
146147

147148
function FilterMatrixDisplayName([array]$matrix, [string]$filter) {
148149
return $matrix | Where-Object { $_ } | ForEach-Object {
149-
if ($_.Name -match $filter) {
150+
if ($_.ContainsKey("Name") -and $_.Name -match $filter) {
150151
return $_
151152
}
152153
}
@@ -168,7 +169,7 @@ function MatchesFilters([hashtable]$entry, [array]$filters) {
168169
# Default all regex checks to go against empty string when keys are missing.
169170
# This simplifies the filter syntax/interface to be regex only.
170171
$value = ""
171-
if ($null -ne $entry -and $entry.parameters.Contains($key)) {
172+
if ($null -ne $entry -and $entry.ContainsKey("parameters") -and $entry.parameters.Contains($key)) {
172173
$value = $entry.parameters[$key]
173174
}
174175
if ($value -notmatch $regex) {
@@ -190,11 +191,34 @@ function ParseFilter([string]$filter) {
190191
}
191192
}
192193

193-
# Importing the JSON as PSCustomObject preserves key ordering,
194-
# whereas ConvertFrom-Json -AsHashtable does not
194+
function GetMatrixConfigFromFile([String] $config)
195+
{
196+
[MatrixConfig]$config = try{
197+
GetMatrixConfigFromJson $config
198+
} catch {
199+
GetMatrixConfigFromYaml $config
200+
}
201+
return $config
202+
}
203+
204+
function GetMatrixConfigFromYaml([String] $yamlConfig)
205+
{
206+
Install-ModuleIfNotInstalled "powershell-yaml" "0.4.1" | Import-Module
207+
# ConvertTo then from json is to make sure the nested values are in PSCustomObject
208+
[MatrixConfig]$config = ConvertFrom-Yaml $yamlConfig -Ordered | ConvertTo-Json -Depth 100 | ConvertFrom-Json
209+
return GetMatrixConfig $config
210+
}
211+
195212
function GetMatrixConfigFromJson([String]$jsonConfig)
196213
{
197214
[MatrixConfig]$config = $jsonConfig | ConvertFrom-Json
215+
return GetMatrixConfig $config
216+
}
217+
218+
# Importing the JSON as PSCustomObject preserves key ordering,
219+
# whereas ConvertFrom-Json -AsHashtable does not
220+
function GetMatrixConfig([MatrixConfig]$config)
221+
{
198222
$config.matrixParameters = @()
199223
$config.displayNamesLookup = @{}
200224
$include = [MatrixParameter[]]@()
@@ -359,7 +383,7 @@ function ProcessImport([MatrixParameter[]]$matrix, [String]$selection, [Array]$n
359383
Write-Error "`$IMPORT path '$importPath' does not exist."
360384
exit 1
361385
}
362-
$importedMatrixConfig = GetMatrixConfigFromJson (Get-Content $importPath)
386+
$importedMatrixConfig = GetMatrixConfigFromFile (Get-Content -Raw $importPath)
363387
$importedMatrix = GenerateMatrix `
364388
-config $importedMatrixConfig `
365389
-selectFromMatrixType $selection `

‎eng/common/scripts/stress-testing/deploy-stress-tests.ps1

+9-1
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,15 @@ param(
2424
[string]$Namespace,
2525

2626
# Override remote stress-test-addons with local on-disk addons for development
27-
[System.IO.FileInfo]$LocalAddonsPath
27+
[System.IO.FileInfo]$LocalAddonsPath,
28+
29+
# Matrix generation parameters
30+
[Parameter(Mandatory=$False)][string]$MatrixFileName,
31+
[Parameter(Mandatory=$False)][string]$MatrixSelection,
32+
[Parameter(Mandatory=$False)][string]$MatrixDisplayNameFilter,
33+
[Parameter(Mandatory=$False)][array]$MatrixFilters,
34+
[Parameter(Mandatory=$False)][array]$MatrixReplace,
35+
[Parameter(Mandatory=$False)][array]$MatrixNonSparseParameters
2836
)
2937

3038
. $PSScriptRoot/stress-test-deployment-lib.ps1

‎eng/common/scripts/stress-testing/find-all-stress-packages.ps1

+26-4
Original file line numberDiff line numberDiff line change
@@ -12,20 +12,42 @@ class StressTestPackageInfo {
1212
[string]$Deployer
1313
}
1414

15+
. $PSScriptRoot/../job-matrix/job-matrix-functions.ps1
16+
. $PSScriptRoot/generate-scenario-matrix.ps1
17+
1518
function FindStressPackages(
1619
[string]$directory,
1720
[hashtable]$filters = @{},
1821
[switch]$CI,
19-
[string]$namespaceOverride
22+
[string]$namespaceOverride,
23+
[string]$MatrixSelection,
24+
[Parameter(Mandatory=$False)][string]$MatrixFileName,
25+
[Parameter(Mandatory=$False)][string]$MatrixDisplayNameFilter,
26+
[Parameter(Mandatory=$False)][array]$MatrixFilters,
27+
[Parameter(Mandatory=$False)][array]$MatrixReplace,
28+
[Parameter(Mandatory=$False)][array]$MatrixNonSparseParameters
2029
) {
2130
# Bare minimum filter for stress tests
2231
$filters['stressTest'] = 'true'
23-
2432
$packages = @()
2533
$chartFiles = Get-ChildItem -Recurse -Filter 'Chart.yaml' $directory
34+
if (!$MatrixFileName) {
35+
$MatrixFileName = '/scenarios-matrix.yaml'
36+
}
2637
foreach ($chartFile in $chartFiles) {
2738
$chart = ParseChart $chartFile
2839
if (matchesAnnotations $chart $filters) {
40+
$matrixFilePath = (Join-Path $chartFile.Directory.FullName $MatrixFileName)
41+
if (Test-Path $matrixFilePath) {
42+
GenerateScenarioMatrix `
43+
-matrixFilePath $matrixFilePath `
44+
-Selection $MatrixSelection `
45+
-DisplayNameFilter $MatrixDisplayNameFilter `
46+
-Filters $MatrixFilters `
47+
-Replace $MatrixReplace `
48+
-NonSparseParameters $MatrixNonSparseParameters
49+
}
50+
2951
$packages += NewStressTestPackageInfo `
3052
-chart $chart `
3153
-chartFile $chartFile `
@@ -80,8 +102,8 @@ function NewStressTestPackageInfo(
80102
Namespace = $namespace.ToLower()
81103
Directory = $chartFile.DirectoryName
82104
ReleaseName = $chart.name
83-
Dockerfile = $chart.annotations.dockerfile
84-
DockerBuildDir = $chart.annotations.dockerbuilddir
105+
Dockerfile = "dockerfile" -in $chart.annotations.keys ? $chart.annotations.dockerfile : $null
106+
DockerBuildDir = "dockerbuilddir" -in $chart.annotations.keys ? $chart.annotations.dockerbuilddir : $null
85107
}
86108
}
87109

‎eng/common/scripts/stress-testing/generate-scenario-matrix.ps1

+86
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
param(
2+
[string]$matrixFilePath,
3+
[string]$Selection,
4+
[Parameter(Mandatory=$False)][string]$DisplayNameFilter,
5+
[Parameter(Mandatory=$False)][array]$Filters,
6+
[Parameter(Mandatory=$False)][array]$Replace,
7+
[Parameter(Mandatory=$False)][array]$NonSparseParameters
8+
)
9+
10+
function GenerateScenarioMatrix(
11+
[string]$matrixFilePath,
12+
[string]$Selection,
13+
[Parameter(Mandatory=$False)][string]$DisplayNameFilter,
14+
[Parameter(Mandatory=$False)][array]$Filters,
15+
[Parameter(Mandatory=$False)][array]$Replace,
16+
[Parameter(Mandatory=$False)][array]$NonSparseParameters
17+
) {
18+
$yamlConfig = Get-Content $matrixFilePath -Raw
19+
20+
$prettyMatrix = &"$PSScriptRoot/../job-matrix/Create-JobMatrix.ps1" `
21+
-ConfigPath $matrixFilePath `
22+
-Selection $Selection `
23+
-DisplayNameFilter $DisplayNameFilter `
24+
-Filters $Filters `
25+
-Replace $Replace `
26+
-NonSparseParameters $NonSparseParameters
27+
Write-Host $prettyMatrix
28+
$prettyMatrix = $prettyMatrix | ConvertFrom-Json
29+
30+
$scenariosMatrix = @()
31+
foreach($permutation in $prettyMatrix.psobject.properties) {
32+
$entry = @{}
33+
$entry.Name = $permutation.Name -replace '_', '-'
34+
$entry.Scenario = $entry.Name
35+
$entry.Remove("Name")
36+
foreach ($param in $permutation.value.psobject.properties) {
37+
$entry.add($param.Name, $param.value)
38+
}
39+
$scenariosMatrix += $entry
40+
}
41+
42+
$valuesYaml = Get-Content -Raw (Join-Path (Split-Path $matrixFilePath) 'values.yaml')
43+
$values = $valuesYaml | ConvertFrom-Yaml -Ordered
44+
if (!$values) {$values = @{}}
45+
46+
if ($values.ContainsKey('Scenarios')) {
47+
throw "Please use matrix generation for stress test scenarios."
48+
}
49+
50+
$values.scenarios = $scenariosMatrix
51+
$values | ConvertTo-Yaml | Out-File -FilePath (Join-Path $matrixFilePath '../generatedValues.yaml')
52+
}
53+
54+
function NewStressTestPackageInfo(
55+
[hashtable]$chart,
56+
[System.IO.FileInfo]$chartFile,
57+
[switch]$CI,
58+
[object]$namespaceOverride
59+
) {
60+
$namespace = if ($namespaceOverride) {
61+
$namespaceOverride
62+
} elseif ($CI) {
63+
$chart.annotations.namespace
64+
} else {
65+
GetUsername
66+
}
67+
68+
return [StressTestPackageInfo]@{
69+
Namespace = $namespace.ToLower()
70+
Directory = $chartFile.DirectoryName
71+
ReleaseName = $chart.name
72+
Dockerfile = $chart.annotations.dockerfile
73+
DockerBuildDir = $chart.annotations.dockerbuilddir
74+
}
75+
}
76+
77+
# Don't call functions when the script is being dot sourced
78+
if ($MyInvocation.InvocationName -ne ".") {
79+
GenerateScenarioMatrix `
80+
-matrixFilePath $matrixFilePath `
81+
-Selection $Selection `
82+
-DisplayNameFilter $DisplayNameFilter `
83+
-Filters $Filters `
84+
-Replace $Replace `
85+
-NonSparseParameters $NonSparseParameters
86+
}

0 commit comments

Comments
 (0)