Skip to content

Commit 10a97e5

Browse files
BigCat20196BigCat20196
authored
Merge pull request #8 from Azure/main
Update 8/4
2 parents 2ef256c + 3d9c147 commit 10a97e5

File tree

903 files changed

+230584
-29055
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

903 files changed

+230584
-29055
lines changed

‎eng/common/docgeneration/Generate-DocIndex.ps1

+19-9
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ function Get-BlobStorage-Artifacts($blobStorageUrl, $blobDirectoryRegex, $blobAr
2727
$blobStorageUrlPageToken = $blobStorageUrl + "&marker=$pageToken"
2828
$resp = Invoke-RestMethod -Method Get -Uri $blobStorageUrlPageToken
2929
}
30-
# Convert to xml documents.
30+
# Convert to xml documents.
3131
$xmlDoc = [xml](removeBomFromString $resp)
3232
foreach ($elem in $xmlDoc.EnumerationResults.Blobs.BlobPrefix) {
3333
# What service return like "dotnet/Azure.AI.Anomalydetector/", needs to fetch out "Azure.AI.Anomalydetector"
@@ -39,8 +39,8 @@ function Get-BlobStorage-Artifacts($blobStorageUrl, $blobDirectoryRegex, $blobAr
3939
} while ($pageToken)
4040
return $returnedArtifacts
4141
}
42-
43-
# The sequence of Bom bytes differs by different encoding.
42+
43+
# The sequence of Bom bytes differs by different encoding.
4444
# The helper function here is only to strip the utf-8 encoding system as it is used by blob storage list api.
4545
# Return the original string if not in BOM utf-8 sequence.
4646
function RemoveBomFromString([string]$bomAwareString) {
@@ -55,8 +55,8 @@ function RemoveBomFromString([string]$bomAwareString) {
5555
}
5656
return $bomAwareString
5757
}
58-
59-
function Get-TocMapping {
58+
59+
function Get-TocMapping {
6060
Param (
6161
[Parameter(Mandatory = $true)] [Object[]] $metadata,
6262
[Parameter(Mandatory = $true)] [String[]] $artifacts
@@ -87,10 +87,10 @@ function Get-TocMapping {
8787
}
8888
$orderServiceMapping[$artifact] = @($serviceName, $displayName)
8989
}
90-
return $orderServiceMapping
90+
return $orderServiceMapping
9191
}
9292

93-
function GenerateDocfxTocContent([Hashtable]$tocContent, [String]$lang) {
93+
function GenerateDocfxTocContent([Hashtable]$tocContent, [String]$lang, [String]$campaignId = "UA-62780441-46") {
9494
LogDebug "Start generating the docfx toc and build docfx site..."
9595

9696
LogDebug "Initializing Default DocFx Site..."
@@ -100,6 +100,15 @@ function GenerateDocfxTocContent([Hashtable]$tocContent, [String]$lang) {
100100
LogDebug "Copying template and configuration..."
101101
New-Item -Path "${DocOutDir}" -Name "templates" -ItemType "directory" -Force
102102
Copy-Item "${DocGenDir}/templates/*" -Destination "${DocOutDir}/templates" -Force -Recurse
103+
104+
$headerTemplateLocation = "${DocOutDir}/templates/matthews/partials/head.tmpl.partial"
105+
106+
if ($campaignId -and (Test-Path $headerTemplateLocation)){
107+
$headerTemplateContent = Get-Content -Path $headerTemplateLocation -Raw
108+
$headerTemplateContent = $headerTemplateContent -replace "GA_CAMPAIGN_ID", $campaignId
109+
Set-Content -Path $headerTemplateLocation -Value $headerTemplateContent -NoNewline
110+
}
111+
103112
Copy-Item "${DocGenDir}/docfx.json" -Destination "${DocOutDir}/" -Force
104113
$YmlPath = "${DocOutDir}/api"
105114
New-Item -Path $YmlPath -Name "toc.yml" -Force
@@ -109,7 +118,7 @@ function GenerateDocfxTocContent([Hashtable]$tocContent, [String]$lang) {
109118
$artifact = $serviceMapping.Key
110119
$serviceName = $serviceMapping.Value[0]
111120
$displayName = $serviceMapping.Value[1]
112-
121+
113122
$fileName = ($serviceName -replace '\s', '').ToLower().Trim()
114123
if ($visitedService.ContainsKey($serviceName)) {
115124
if ($displayName) {
@@ -145,7 +154,7 @@ function GenerateDocfxTocContent([Hashtable]$tocContent, [String]$lang) {
145154
& $($DocFx) build "${DocOutDir}/docfx.json"
146155
# The line below is used for testing in local
147156
#docfx build "${DocOutDir}/docfx.json"
148-
Copy-Item "${DocGenDir}/assets/logo.svg" -Destination "${DocOutDir}/_site/" -Force
157+
Copy-Item "${DocGenDir}/assets/logo.svg" -Destination "${DocOutDir}/_site/" -Force
149158
}
150159

151160
function UpdateDocIndexFiles {
@@ -166,6 +175,7 @@ function UpdateDocIndexFiles {
166175
# Update main.js package regex and replacement
167176
$mainJsContent = $mainJsContent -replace "var PACKAGE_REGEX = ''", "var PACKAGE_REGEX = $packageRegex"
168177
$mainJsContent = $mainJsContent -replace "var PACKAGE_REPLACEMENT = ''", "var PACKAGE_REPLACEMENT = `"$regexReplacement`""
178+
169179
Set-Content -Path $MainJsPath -Value $mainJsContent -NoNewline
170180
}
171181

‎eng/common/docgeneration/templates/matthews/partials/head.tmpl.partial

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
<head>
2+
<meta charset="utf-8">
3+
<meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
4+
<title>{{#title}}{{title}}{{/title}}{{^title}}{{>partials/title}}{{/title}} {{#_appTitle}}| {{_appTitle}} {{/_appTitle}}</title>
5+
<meta name="viewport" content="width=device-width">
6+
<meta name="title" content="{{#title}}{{title}}{{/title}}{{^title}}{{>partials/title}}{{/title}} {{#_appTitle}}| {{_appTitle}} {{/_appTitle}}">
7+
<meta name="generator" content="docfx {{_docfxVersion}}">
8+
{{#_description}}<meta name="description" content="{{_description}}">{{/_description}}
9+
<link rel="shortcut icon" href="{{_rel}}{{{_appFaviconPath}}}{{^_appFaviconPath}}favicon.ico{{/_appFaviconPath}}">
10+
<link rel="stylesheet" href="{{_rel}}styles/docfx.vendor.css">
11+
<link rel="stylesheet" href="{{_rel}}styles/docfx.css">
12+
<link rel="stylesheet" href="{{_rel}}styles/main.css">
13+
<meta property="docfx:navrel" content="{{_navRel}}">
14+
<meta property="docfx:tocrel" content="{{_tocRel}}">
15+
{{#_noindex}}<meta name="searchOption" content="noindex">{{/_noindex}}
16+
{{#_enableSearch}}<meta property="docfx:rel" content="{{_rel}}">{{/_enableSearch}}
17+
{{#_enableNewTab}}<meta property="docfx:newtab" content="true">{{/_enableNewTab}}
18+
19+
<!-- Global site tag (gtag.js) - Google Analytics -->
20+
<script async src="https://www.googletagmanager.com/gtag/js?id=GA_CAMPAIGN_ID"></script>
21+
<script>
22+
window.dataLayer = window.dataLayer || [];
23+
function gtag(){dataLayer.push(arguments);}
24+
gtag('js', new Date());
25+
26+
gtag('config', 'GA_CAMPAIGN_ID');
27+
</script>
28+
</head>

‎eng/common/scripts/Create-APIReview.ps1

+14-2
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ Param (
1515
)
1616

1717
# Submit API review request and return status whether current revision is approved or pending or failed to create review
18-
function Submit-APIReview($packagename, $filePath, $uri, $apiKey, $apiLabel)
18+
function Submit-APIReview($packagename, $filePath, $uri, $apiKey, $apiLabel, $releaseStatus)
1919
{
2020
$multipartContent = [System.Net.Http.MultipartFormDataContent]::new()
2121
$FileStream = [System.IO.FileStream]::new($filePath, [System.IO.FileMode]::Open)
@@ -33,6 +33,17 @@ function Submit-APIReview($packagename, $filePath, $uri, $apiKey, $apiLabel)
3333
$StringContent = [System.Net.Http.StringContent]::new($apiLabel)
3434
$StringContent.Headers.ContentDisposition = $stringHeader
3535
$multipartContent.Add($stringContent)
36+
Write-Host "Request param, label: $apiLabel"
37+
38+
if ($releaseStatus -and ($releaseStatus -ne "Unreleased"))
39+
{
40+
$compareAllParam = [System.Net.Http.Headers.ContentDispositionHeaderValue]::new("form-data")
41+
$compareAllParam.Name = "compareAllRevisions"
42+
$compareAllParamContent = [System.Net.Http.StringContent]::new($true)
43+
$compareAllParamContent.Headers.ContentDisposition = $compareAllParam
44+
$multipartContent.Add($compareAllParamContent)
45+
Write-Host "Request param, compareAllRevisions: true"
46+
}
3647

3748
$headers = @{
3849
"ApiKey" = $apiKey;
@@ -103,13 +114,14 @@ if ($packages)
103114

104115
Write-Host "Version: $($version)"
105116
Write-Host "SDK Type: $($pkgInfo.SdkType)"
117+
Write-Host "Release Status: $($pkgInfo.ReleaseStatus)"
106118

107119
# Run create review step only if build is triggered from main branch or if version is GA.
108120
# This is to avoid invalidating review status by a build triggered from feature branch
109121
if ( ($SourceBranch -eq $DefaultBranch) -or (-not $version.IsPrerelease))
110122
{
111123
Write-Host "Submitting API Review for package $($pkg)"
112-
$respCode = Submit-APIReview -packagename $pkg -filePath $pkgPath -uri $APIViewUri -apiKey $APIKey -apiLabel $APILabel
124+
$respCode = Submit-APIReview -packagename $pkg -filePath $pkgPath -uri $APIViewUri -apiKey $APIKey -apiLabel $APILabel -releaseStatus $pkgInfo.ReleaseStatus
113125
Write-Host "HTTP Response code: $($respCode)"
114126
# HTTP status 200 means API is in approved status
115127
if ($respCode -eq '200')

‎eng/common/scripts/stress-testing/deploy-stress-tests.ps1

+174
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,174 @@
1+
[CmdletBinding(DefaultParameterSetName = 'Default')]
2+
param(
3+
[string]$SearchDirectory,
4+
[hashtable]$Filters,
5+
[string]$Environment,
6+
[string]$Repository,
7+
[switch]$PushImages,
8+
[string]$ClusterGroup,
9+
[string]$DeployId,
10+
11+
[Parameter(ParameterSetName = 'DoLogin', Mandatory = $true)]
12+
[switch]$Login,
13+
14+
[Parameter(ParameterSetName = 'DoLogin')]
15+
[string]$Subscription
16+
)
17+
18+
$ErrorActionPreference = 'Stop'
19+
20+
. $PSScriptRoot/find-all-stress-packages.ps1
21+
$FailedCommands = New-Object Collections.Generic.List[hashtable]
22+
23+
if (!(Get-Module powershell-yaml)) {
24+
Install-Module -Name powershell-yaml -RequiredVersion 0.4.1 -Force -Scope CurrentUser
25+
}
26+
27+
# Powershell does not (at time of writing) treat exit codes from external binaries
28+
# as cause for stopping execution, so do this via a wrapper function.
29+
# See https://github.com/PowerShell/PowerShell-RFC/pull/277
30+
function Run() {
31+
Write-Host "`n==> $args`n" -ForegroundColor Green
32+
$command, $arguments = $args
33+
& $command $arguments
34+
if ($LASTEXITCODE) {
35+
Write-Error "Command '$args' failed with code: $LASTEXITCODE" -ErrorAction 'Continue'
36+
$FailedCommands.Add(@{ command = "$args"; code = $LASTEXITCODE })
37+
}
38+
}
39+
40+
function RunOrExitOnFailure() {
41+
run @args
42+
if ($LASTEXITCODE) {
43+
exit $LASTEXITCODE
44+
}
45+
}
46+
47+
function Login([string]$subscription, [string]$clusterGroup, [boolean]$pushImages) {
48+
Write-Host "Logging in to subscription, cluster and container registry"
49+
az account show *> $null
50+
if ($LASTEXITCODE) {
51+
RunOrExitOnFailure az login --allow-no-subscriptions
52+
}
53+
54+
$clusterName = (az aks list -g $clusterGroup -o json| ConvertFrom-Json).name
55+
56+
RunOrExitOnFailure az aks get-credentials `
57+
-n "$clusterName" `
58+
-g "$clusterGroup" `
59+
--subscription "$subscription" `
60+
--overwrite-existing
61+
62+
if ($pushImages) {
63+
$registry = (az acr list -g $clusterGroup -o json | ConvertFrom-Json).name
64+
RunOrExitOnFailure az acr login -n $registry
65+
}
66+
}
67+
68+
function DeployStressTests(
69+
[string]$searchDirectory = '.',
70+
[hashtable]$filters = @{},
71+
[string]$environment = 'test',
72+
[string]$repository = 'images',
73+
[boolean]$pushImages = $false,
74+
[string]$clusterGroup = 'rg-stress-test-cluster-',
75+
[string]$deployId = 'local',
76+
[string]$subscription = 'Azure SDK Test Resources'
77+
) {
78+
if ($PSCmdlet.ParameterSetName -eq 'DoLogin') {
79+
Login $subscription $clusterGroup $pushImages
80+
}
81+
82+
RunOrExitOnFailure helm repo add stress-test-charts https://stresstestcharts.blob.core.windows.net/helm/
83+
Run helm repo update
84+
if ($LASTEXITCODE) { return $LASTEXITCODE }
85+
86+
$pkgs = FindStressPackages $searchDirectory $filters
87+
Write-Host "" "Found $($pkgs.Length) stress test packages:"
88+
Write-Host $pkgs.Directory ""
89+
foreach ($pkg in $pkgs) {
90+
Write-Host "Deploying stress test at '$($pkg.Directory)'"
91+
DeployStressPackage $pkg $deployId $environment $repository $pushImages
92+
}
93+
94+
Write-Host "Releases deployed by $deployId"
95+
Run helm list --all-namespaces -l deployId=$deployId
96+
97+
if ($FailedCommands) {
98+
Write-Warning "The following commands failed:"
99+
foreach ($cmd in $FailedCommands) {
100+
Write-Error "'$($cmd.command)' failed with code $($cmd.code)" -ErrorAction 'Continue'
101+
}
102+
exit 1
103+
}
104+
}
105+
106+
function DeployStressPackage(
107+
[object]$pkg,
108+
[string]$deployId,
109+
[string]$environment,
110+
[string]$repository,
111+
[boolean]$pushImages
112+
) {
113+
$registry = (az acr list -g $clusterGroup -o json | ConvertFrom-Json).name
114+
if (!$registry) {
115+
Write-Host "Could not find container registry in resource group $clusterGroup"
116+
exit 1
117+
}
118+
119+
if ($pushImages) {
120+
Run helm dependency update $pkg.Directory
121+
if ($LASTEXITCODE) { return $LASTEXITCODE }
122+
123+
$dockerFiles = Get-ChildItem "$($pkg.Directory)/Dockerfile*"
124+
foreach ($dockerFile in $dockerFiles) {
125+
# Infer docker image name from parent directory name, if file is named `Dockerfile`
126+
# or from suffix, is file is named like `Dockerfile.myimage` (for multiple dockerfiles).
127+
$prefix, $imageName = $dockerFile.Name.Split(".")
128+
if (!$imageName) {
129+
$imageName = $dockerFile.Directory.Name
130+
}
131+
$imageTag = "${registry}.azurecr.io/$($repository.ToLower())/$($imageName):$deployId"
132+
Write-Host "Building and pushing stress test docker image '$imageTag'"
133+
Run docker build -t $imageTag -f $dockerFile.FullName $dockerFile.DirectoryName
134+
if ($LASTEXITCODE) { return $LASTEXITCODE }
135+
Run docker push $imageTag
136+
if ($LASTEXITCODE) {
137+
if ($PSCmdlet.ParameterSetName -ne 'DoLogin') {
138+
Write-Warning "If docker push is failing due to authentication issues, try calling this script with '-Login'"
139+
}
140+
return $LASTEXITCODE
141+
}
142+
}
143+
}
144+
145+
Write-Host "Creating namespace $($pkg.Namespace) if it does not exist..."
146+
kubectl create namespace $pkg.Namespace --dry-run=client -o yaml | kubectl apply -f -
147+
148+
Write-Host "Installing or upgrading stress test $($pkg.ReleaseName) from $($pkg.Directory)"
149+
Run helm upgrade $pkg.ReleaseName $pkg.Directory `
150+
-n $pkg.Namespace `
151+
--install `
152+
--set repository=$registry.azurecr.io/$repository `
153+
--set tag=$deployId `
154+
--set stress-test-addons.env=$environment
155+
if ($LASTEXITCODE) {
156+
# Issues like 'UPGRADE FAILED: another operation (install/upgrade/rollback) is in progress'
157+
# can be the result of cancelled `upgrade` operations (e.g. ctrl-c).
158+
# See https://github.com/helm/helm/issues/4558
159+
Write-Warning "The issue may be fixable by first running 'helm rollback -n $($pkg.Namespace) $($pkg.ReleaseName)'"
160+
return $LASTEXITCODE
161+
}
162+
163+
# Helm 3 stores release information in kubernetes secrets. The only way to add extra labels around
164+
# specific releases (thereby enabling filtering on `helm list`) is to label the underlying secret resources.
165+
# There is not currently support for setting these labels via the helm cli.
166+
$helmReleaseConfig = kubectl get secrets `
167+
-n $pkg.Namespace `
168+
-l status=deployed,name=$($pkg.ReleaseName) `
169+
-o jsonpath='{.items[0].metadata.name}'
170+
171+
Run kubectl label secret -n $pkg.Namespace --overwrite $helmReleaseConfig deployId=$deployId
172+
}
173+
174+
DeployStressTests @PSBoundParameters

‎eng/common/scripts/stress-testing/find-all-stress-packages.ps1

+53
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
param(
2+
[string]$searchDirectory = '.',
3+
[hashtable]$filters = @{}
4+
)
5+
6+
class StressTestPackageInfo {
7+
[string]$Namespace
8+
[string]$Directory
9+
[string]$ReleaseName
10+
}
11+
12+
function FindStressPackages([string]$directory, [hashtable]$filters = @{}) {
13+
# Bare minimum filter for stress tests
14+
$filters['stressTest'] = 'true'
15+
16+
$packages = @()
17+
$chartFiles = Get-ChildItem -Recurse -Filter 'Chart.yaml' $directory
18+
foreach ($chartFile in $chartFiles) {
19+
$chart = ParseChart $chartFile
20+
if (matchesAnnotations $chart $filters) {
21+
$packages += NewStressTestPackageInfo $chart $chartFile
22+
}
23+
}
24+
25+
return $packages
26+
}
27+
28+
function ParseChart([string]$chartFile) {
29+
return ConvertFrom-Yaml (Get-Content -Raw $chartFile)
30+
}
31+
32+
function MatchesAnnotations([hashtable]$chart, [hashtable]$filters) {
33+
foreach ($filter in $filters.GetEnumerator()) {
34+
if (!$chart.annotations -or $chart.annotations[$filter.Key] -ne $filter.Value) {
35+
return $false
36+
}
37+
}
38+
39+
return $true
40+
}
41+
42+
function NewStressTestPackageInfo([hashtable]$chart, [System.IO.FileInfo]$chartFile) {
43+
return [StressTestPackageInfo]@{
44+
Namespace = $chart.annotations.namespace
45+
Directory = $chartFile.DirectoryName
46+
ReleaseName = $chart.name
47+
}
48+
}
49+
50+
# Don't call functions when the script is being dot sourced
51+
if ($MyInvocation.InvocationName -ne ".") {
52+
FindStressPackages $searchDirectory $filters
53+
}

0 commit comments

Comments
 (0)