-
Notifications
You must be signed in to change notification settings - Fork 3.1k
/
Copy pathlinux-gpu-tensorrt-daily-perf-pipeline.yml
199 lines (161 loc) · 9.2 KB
/
linux-gpu-tensorrt-daily-perf-pipeline.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
parameters:
- name: PostToDashboard
displayName: Post EP Perf results to Dashboard
type: boolean
default: true
- name: TrtVersion
displayName: TensorRT Version
type: string
default: 10.8_cuda12.6_cudnn9
values:
- 10.8_cuda11.8_cudnn8
- 10.8_cuda12.6_cudnn9
- BIN
- name: UseTensorrtOssParser
displayName: Use TensorRT-OSS Parser (not compatible with BIN)
type: boolean
default: false
- name: ModelGroups
type: object
default:
- "onnx-zoo-models"
- "partner-models"
- name: MemTest
displayName: Run Memory and Concurrency Test
type: boolean
default: true
- name: ConcurrencyTest
displayName: Specifies the number of concurrency model test to invoke simultaneously
type: string
default: 2
- name: TrtEPOptions
displayName: TensorRT EP options
type: object
default:
- trt_max_workspace_size=4294967296
- trt_engine_cache_enable=True
- name: CUDAEPOptions
displayName: CUDA EP options
type: object
default: []
jobs:
- job: 'Onnxruntime_Linux_GPU_TensorRT_Perf'
workspace:
clean: all
timeoutInMinutes: 3000
pool: 'Onnxruntime-Linux-GPU-TensorRT-Perf'
variables:
- name: branchName
${{ if not(eq(variables['ortBranchOverride'], '')) }}:
value: $(ortBranchOverride)
${{ else }}:
value: $[ replace(variables['Build.SourceBranch'], 'refs/heads/', '') ]
- name: trtVersion
${{ if eq(parameters.TrtVersion, 'BIN') }}:
value: $(tarTrtVersion)
${{ else }}:
value: ${{ parameters.TrtVersion }}
- name: trtEPOptionsArg
${{ if not(eq(length(parameters.TrtEPOptions), 0)) }}:
value: --trt_ep_options ${{ join(',',parameters.TrtEPOptions) }}
- name: cudaEPOptionsArg
${{ if not(eq(length(parameters.CUDAEPOptions), 0)) }}:
value: --cuda_ep_options ${{ join(',',parameters.CUDAEPOptions) }}
- name: optional_arguments
value: -a "-a -z -g $(optimizeGraph) -b $(bindInputs) $(trtEPOptionsArg) $(cudaEPOptionsArg)"
- name: image
value: ort-image-$(Build.BuildId)
- name: parser
${{ if eq(parameters.UseTensorrtOssParser, true) }}:
value: --use_tensorrt_oss_parser $(parameters.UseTensorrtOssParser)
steps:
- ${{ if and(eq(parameters.TrtVersion, 'BIN'), eq(parameters.UseTensorrtOssParser, false)) }}:
- script: 'ls -al $(trtBinsDir)'
displayName: 'Show available TensorRT .tar.gz packages'
- script: 'cp $(trtBinsDir)/TensorRT-$(trtVersion).Linux.x86_64-gnu.cuda-$(tarCudaVersion).tar.gz $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/build/'
displayName: 'Copy TensorRT .tar.gz package into Docker build directory'
- script: 'python3 $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/build/build_image.py -r $(Build.SourcesDirectory) -i $(image) -b $(branchName) -t $(trtVersion) -a 75 --install_bin --tar_cuda_version=$(tarCudaVersion) --trt_bins_dir=.'
displayName: 'Install TensorRT $(tarTrtVersion) from binaries and build latest ORT Image'
workingDirectory: '$(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/build'
# Build ORT with TensorRT built-in parser
- ${{ if and(ne(parameters.TrtVersion, 'BIN'), eq(parameters.UseTensorrtOssParser, false)) }}:
- script: 'python3 $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/build/build_image.py -r $(Build.SourcesDirectory) -i $(image) -b $(branchName) -t $(trtVersion) -a 75'
displayName: 'Build latest ORT Image with TensorRT built-in parser'
workingDirectory: '$(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/build'
# Build ORT with certain version of TensorRT OSS parser
- ${{ if and(ne(parameters.TrtVersion, 'BIN'), eq(parameters.UseTensorrtOssParser, true), ne(variables.onnxTensorrtCommitId, '')) }}:
- script: 'python3 $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/build/build_image.py -r $(Build.SourcesDirectory) -i $(image) -b $(branchName) -t $(trtVersion) -a 75 -o $(onnxTensorrtCommitId) --use_tensorrt_oss_parser'
displayName: 'Build latest ORT Image with TensorRT OSS parser and commit id $(onnxTensorrtCommitId)'
workingDirectory: '$(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/build'
# Build ORT with default version of TensorRT OSS parser
- ${{ if and(ne(parameters.TrtVersion, 'BIN'), eq(parameters.UseTensorrtOssParser, true), eq(variables.onnxTensorrtCommitId, '')) }}:
- script: 'python3 $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/build/build_image.py -r $(Build.SourcesDirectory) -i $(image) -b $(branchName) -t $(trtVersion) -a 75 --use_tensorrt_oss_parser'
displayName: 'Build latest ORT Image with TensorRT OSS parser'
workingDirectory: '$(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/build'
- ${{ if eq(parameters.MemTest, true) }}:
- script: '$(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/mem_test/run_mem_test_docker.sh -d $(image) -p $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/mem_test/ -w /code/ -l false -c ${{ parameters.ConcurrencyTest }}'
displayName: 'Run Memory Test and Concurrency Test'
workingDirectory: '$(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/mem_test/'
- ${{ each option in parameters.ModelGroups }}:
- script: '$(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/run_perf_docker.sh -d $(image) -p $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf -v $(modelVolume) -b true -o ${{option}} -m $(${{option}}) -e "$(epList)" $(optional_arguments)'
displayName: '${{option}} perf'
workingDirectory: '$(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/'
# Prepare and Publish Artifacts
- script: 'mkdir $(Build.SourcesDirectory)/Artifact'
displayName: 'Prepare Artifacts Directory'
workingDirectory: '$(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/'
condition: always()
- ${{ if not(eq(length(parameters.ModelGroups), 0)) }}:
- script: 'cp -r $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/result/ $(Build.SourcesDirectory)/Artifact'
displayName: 'Copy Artifacts'
workingDirectory: '$(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/'
- ${{ if eq(parameters.MemTest, true) }}:
- script: 'cp -r $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/mem_test/build/result $(Build.SourcesDirectory)/Artifact/result_mem_test'
displayName: 'Copy Artifacts'
workingDirectory: '$(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/'
condition: always()
- script: 'python3 -m pip install pandas azure-kusto-data[pandas] azure-kusto-ingest[pandas] coloredlogs'
displayName: 'Install dashboard dependencies'
- script: |
az --version || {
echo "Azure CLI not found, installing..."
curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
}
displayName: 'Check and Install Azure CLI'
- task: AzureCLI@2
displayName: 'Parse Memory & Concurrency Test Records and Sync'
inputs:
azureSubscription: AIInfraBuildOnnxRuntimeOSS
scriptLocation: inlineScript
scriptType: bash
inlineScript: |
short_hash=$(git rev-parse --short HEAD) &&
commit_date=$(git log -1 --date=iso-strict --pretty=format:%cd) &&
python3 $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/parse_mem_concurrency_test.py -r $(Build.SourcesDirectory)/Artifact/result_mem_test -c $short_hash -d $commit_date -u "$(reportUrl)?buildId=$(Build.BuildId)" -t $(trtVersion) -b $(branchName) --kusto_conn $(kustoConn) --database $(database) $(parser)
- ${{ if eq(parameters.PostToDashboard, true) }}:
- script: 'python3 -m pip install pandas azure-kusto-data[pandas] azure-kusto-ingest[pandas] coloredlogs'
displayName: 'Install dashboard dependencies'
- script: |
az --version || {
echo "Azure CLI not found, installing..."
curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
}
displayName: 'Check and Install Azure CLI'
- task: AzureCLI@2
displayName: 'Azure CLI Post to Dashboard'
inputs:
azureSubscription: AIInfraBuildOnnxRuntimeOSS
scriptLocation: inlineScript
scriptType: bash
inlineScript: |
short_hash=$(git rev-parse --short HEAD) &&
commit_date=$(git log -1 --date=iso-strict --pretty=format:%cd) &&
python3 $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/post.py -r $(Build.SourcesDirectory)/Artifact/result -c $short_hash -d $commit_date -u "$(reportUrl)?buildId=$(Build.BuildId)" -t $(trtVersion) -b $(branchName) --kusto_conn $(kustoConn) --database $(database) $(parser)
- task: PublishBuildArtifacts@1
inputs:
pathtoPublish: '$(Build.SourcesDirectory)/Artifact'
artifactName: 'result-$(Build.BuildNumber)'
- template: templates/component-governance-component-detection-steps.yml
parameters :
condition : 'succeeded'
- template: templates/clean-agent-build-directory-step.yml