|
28 | 28 | import os |
29 | 29 | import tempfile |
30 | 30 | import zipfile |
| 31 | +import re |
31 | 32 | from glob import glob |
| 33 | +from pathlib import Path |
32 | 34 |
|
33 | 35 | import mx |
34 | 36 | import mx_benchmark |
@@ -264,6 +266,204 @@ def successPatterns(self): |
264 | 266 | mx_benchmark.add_bm_suite(RenaissanceNativeImageBenchmarkSuite()) |
265 | 267 |
|
266 | 268 |
|
| 269 | +class BaristaNativeImageBenchmarkSuite(mx_sdk_benchmark.BaristaBenchmarkSuite, mx_sdk_benchmark.NativeImageBenchmarkMixin, mx_sdk_benchmark.NativeImageBundleBasedBenchmarkMixin): |
| 270 | + """Native Image variant of the Barista benchmark suite implementation. A collection of microservice workloads running in native execution mode on the Barista harness. |
| 271 | +
|
| 272 | + The run arguments are passed to the Barista harness. |
| 273 | + If you want to run something like `hwloc-bind` or `taskset` prefixed before the app image, you should use the '--cmd-app-prefix' Barista harness option. |
| 274 | + If you want to pass options to the app image, you should use the '--app-args' Barista harness option. |
| 275 | + """ |
| 276 | + def __init__(self, custom_harness_command: mx_benchmark.CustomHarnessCommand = None): |
| 277 | + if custom_harness_command is None: |
| 278 | + custom_harness_command = BaristaNativeImageBenchmarkSuite.BaristaNativeImageCommand() |
| 279 | + super().__init__(custom_harness_command) |
| 280 | + self._application_nibs = {} |
| 281 | + # because of an issue in handling image build args in the intended order [GR-58214] |
| 282 | + # we need the image name that is set inside the nib |
| 283 | + self._application_fixed_image_names = {} |
| 284 | + |
| 285 | + def name(self): |
| 286 | + return "barista-native-image" |
| 287 | + |
| 288 | + def benchSuiteName(self, bmSuiteArgs=None): |
| 289 | + return "barista" |
| 290 | + |
| 291 | + def benchmarkName(self): |
| 292 | + return self.context.benchmark |
| 293 | + |
| 294 | + def application_nib(self): |
| 295 | + if self.benchmarkName() not in self._application_nibs: |
| 296 | + # Run subprocess retrieving the application nib from the Barista 'build' script |
| 297 | + out = mx.OutputCapture() |
| 298 | + mx.run([self.baristaBuilderPath(), "--get-nib", self.baristaHarnessBenchmarkName()], out=out) |
| 299 | + # Capture the application nib from the Barista 'build' script output |
| 300 | + nib_pattern = r"application nib file path is: ([^\n]+)\n" |
| 301 | + nib_match = re.search(nib_pattern, out.data) |
| 302 | + if not nib_match: |
| 303 | + raise ValueError(f"Could not extract the nib file path from the command output! Expected to match pattern {repr(nib_pattern)}.") |
| 304 | + # Cache for future access |
| 305 | + self._application_nibs[self.benchmarkName()] = nib_match.group(1) |
| 306 | + # Try to capture the fixed image name from the Barista 'build' script output |
| 307 | + fixed_image_name_pattern = r"fixed image name is: ([^\n]+)\n" |
| 308 | + fixed_image_name_match = re.search(fixed_image_name_pattern, out.data) |
| 309 | + # Cache fixed image name, if present |
| 310 | + if fixed_image_name_match: |
| 311 | + self._application_fixed_image_names[self.benchmarkName()] = fixed_image_name_match.group(1) |
| 312 | + return self._application_nibs[self.benchmarkName()] |
| 313 | + |
| 314 | + def application_fixed_image_name(self): |
| 315 | + self.application_nib() |
| 316 | + return self._application_fixed_image_names.get(self.benchmarkName(), None) |
| 317 | + |
| 318 | + def applicationDist(self): |
| 319 | + return Path(self.application_nib()).parent |
| 320 | + |
| 321 | + def uses_bundles(self): |
| 322 | + return True |
| 323 | + |
| 324 | + def createCommandLineArgs(self, benchmarks, bmSuiteArgs): |
| 325 | + # Pass the VM options, BaristaNativeImageCommand will form the final command. |
| 326 | + return self.vmArgs(bmSuiteArgs) |
| 327 | + |
| 328 | + def extra_jvm_arg(self, benchmark, args): |
| 329 | + # Added by BaristaNativeImageCommand |
| 330 | + return [] |
| 331 | + |
| 332 | + def extra_agent_run_arg(self, benchmark, args, image_run_args): |
| 333 | + # Added by BaristaNativeImageCommand |
| 334 | + return [] |
| 335 | + |
| 336 | + def extra_profile_run_arg(self, benchmark, args, image_run_args, should_strip_run_args): |
| 337 | + # Added by BaristaNativeImageCommand |
| 338 | + return [] |
| 339 | + |
| 340 | + def extra_run_arg(self, benchmark, args, image_run_args): |
| 341 | + # Added by BaristaNativeImageCommand |
| 342 | + return [] |
| 343 | + |
| 344 | + def run(self, benchmarks, bmSuiteArgs) -> mx_benchmark.DataPoints: |
| 345 | + return self.intercept_run(super(), benchmarks, bmSuiteArgs) |
| 346 | + |
| 347 | + def ensure_image_is_at_desired_location(self, bmSuiteArgs): |
| 348 | + if self.stages_info.requested_stage.is_image() and self.application_fixed_image_name() is not None: |
| 349 | + # Because of an issue in handling image build args in the intended order [GR-58214] |
| 350 | + # we need to move the image from the path that is set inside the nib to the path expected by our vm. |
| 351 | + # This code has no effect if the image is already at the desired location. |
| 352 | + vm = self.get_vm_registry().get_vm_from_suite_args(bmSuiteArgs) |
| 353 | + if vm.stages_info.should_produce_datapoints(mx_sdk_benchmark.Stage.INSTRUMENT_IMAGE): |
| 354 | + desired_image_path = vm.config.instrumented_image_path |
| 355 | + elif vm.stages_info.should_produce_datapoints(mx_sdk_benchmark.Stage.IMAGE): |
| 356 | + desired_image_path = vm.config.image_path |
| 357 | + else: |
| 358 | + return |
| 359 | + actual_image_path = desired_image_path.parent / self.application_fixed_image_name() |
| 360 | + if actual_image_path.is_file() and not desired_image_path.is_file(): |
| 361 | + mx.move(actual_image_path, desired_image_path) |
| 362 | + |
| 363 | + def runAndReturnStdOut(self, benchmarks, bmSuiteArgs): |
| 364 | + retcode, out, dims = super().runAndReturnStdOut(benchmarks, bmSuiteArgs) |
| 365 | + self.ensure_image_is_at_desired_location(bmSuiteArgs) |
| 366 | + return retcode, out, dims |
| 367 | + |
| 368 | + class BaristaNativeImageCommand(mx_sdk_benchmark.BaristaBenchmarkSuite.BaristaCommand): |
| 369 | + """Maps the command produced by NativeImageVM into a command tailored for the Barista harness. |
| 370 | + """ |
| 371 | + def _short_load_testing_phases(self): |
| 372 | + """Configures the main barista load-testing phases to be quite short. |
| 373 | +
|
| 374 | + Useful for the `agent` and `instrument-run` stages. |
| 375 | + """ |
| 376 | + return [ |
| 377 | + "--warmup-iteration-count", "1", |
| 378 | + "--warmup-duration", "5", |
| 379 | + "--throughput-iteration-count", "0", |
| 380 | + "--latency-iteration-count", "0", |
| 381 | + ] |
| 382 | + |
| 383 | + def _get_built_app_image(self, suite, stage): |
| 384 | + """Retrieves the path to the app image built in the previous stage. |
| 385 | +
|
| 386 | + In the case of `instrument-run`, retrieves the image built during `instrument-image`. |
| 387 | + In the case of `run`, retrieves the image built during `image`. |
| 388 | + """ |
| 389 | + vm = suite.context.vm |
| 390 | + if stage == mx_sdk_benchmark.Stage.INSTRUMENT_RUN: |
| 391 | + return vm.config.instrumented_image_path |
| 392 | + else: |
| 393 | + return vm.config.image_path |
| 394 | + |
| 395 | + def produce_JVM_harness_command(self, cmd, suite): |
| 396 | + """Maps a JVM command into a command tailored for the Barista harness. |
| 397 | +
|
| 398 | + Utilizes the implementation of the ``mx_sdk_benchmark.BaristaBenchmarkSuite.BaristaCommand`` base class |
| 399 | + """ |
| 400 | + return super().produceHarnessCommand(cmd, suite) |
| 401 | + |
| 402 | + def produceHarnessCommand(self, cmd, suite): |
| 403 | + """Maps a NativeImageVM command into a command tailored for the Barista harness. |
| 404 | +
|
| 405 | + This method is invoked only in the `agent`, `instrument-run` and `run` stages, because hooks are |
| 406 | + only applied in these stages (defined in ``NativeImageBenchmarkMixin.run_stage``). |
| 407 | + In the case of the `agent` stage, relies on the parent ``BaristaCommand`` class for the mapping. |
| 408 | +
|
| 409 | + :param list[str] cmd: NativeImageVM command to be mapped. |
| 410 | + :param BaristaNativeImageBenchmarkSuite suite: Barista benchmark suite running the benchmark on the Barista harness. |
| 411 | + :return: Command tailored for the Barista harness. |
| 412 | + :rtype: list[str] |
| 413 | + """ |
| 414 | + if not isinstance(suite, BaristaNativeImageBenchmarkSuite): |
| 415 | + raise TypeError(f"Expected an instance of {BaristaNativeImageBenchmarkSuite.__name__}, instead got an instance of {suite.__class__.__name__}") |
| 416 | + |
| 417 | + stage = suite.stages_info.requested_stage |
| 418 | + if stage == mx_sdk_benchmark.Stage.AGENT: |
| 419 | + # BaristaCommand works for agent stage, since it's a JVM stage |
| 420 | + cmd = self.produce_JVM_harness_command(cmd, suite) |
| 421 | + # Make agent run short |
| 422 | + cmd += self._short_load_testing_phases() |
| 423 | + # Add explicit agent stage args |
| 424 | + cmd += mx_sdk_benchmark.parse_prefixed_args("-Dnative-image.benchmark.extra-jvm-arg=", suite.context.bmSuiteArgs) |
| 425 | + cmd += mx_sdk_benchmark.parse_prefixed_args("-Dnative-image.benchmark.extra-agent-run-arg=", suite.context.bmSuiteArgs) |
| 426 | + return cmd |
| 427 | + |
| 428 | + # Extract app image options and command prefix from the NativeImageVM command |
| 429 | + app_image = str(self._get_built_app_image(suite, stage)) |
| 430 | + try: |
| 431 | + index_of_app_image = cmd.index(app_image) |
| 432 | + except: |
| 433 | + mx.log_error(f"Could not find app image '{app_image}' in {cmd}") |
| 434 | + raise |
| 435 | + nivm_cmd_prefix = cmd[:index_of_app_image] |
| 436 | + nivm_app_options = cmd[index_of_app_image + 1:] |
| 437 | + |
| 438 | + # Get bench name and workload to use in the barista harness - we might have custom named benchmarks that need to be mapped |
| 439 | + barista_bench_name = suite.baristaHarnessBenchmarkName() |
| 440 | + barista_workload = suite.baristaHarnessBenchmarkWorkload() |
| 441 | + |
| 442 | + # Provide image built in the previous stage to the Barista harnesss using the `--app-executable` option |
| 443 | + ni_barista_cmd = [suite.baristaHarnessPath(), "--mode", "native", "--app-executable", app_image] |
| 444 | + if barista_workload is not None: |
| 445 | + ni_barista_cmd.append(f"--config={barista_workload}") |
| 446 | + ni_barista_cmd += suite.runArgs(suite.context.bmSuiteArgs) |
| 447 | + ni_barista_cmd += mx_sdk_benchmark.parse_prefixed_args("-Dnative-image.benchmark.extra-jvm-arg=", suite.context.bmSuiteArgs) |
| 448 | + if stage == mx_sdk_benchmark.Stage.INSTRUMENT_RUN: |
| 449 | + # Make instrument run short |
| 450 | + ni_barista_cmd += self._short_load_testing_phases() |
| 451 | + # Add explicit instrument stage args |
| 452 | + ni_barista_cmd += mx_sdk_benchmark.parse_prefixed_args("-Dnative-image.benchmark.extra-profile-run-arg=", suite.context.bmSuiteArgs) or mx_sdk_benchmark.parse_prefixed_args("-Dnative-image.benchmark.extra-run-arg=", suite.context.bmSuiteArgs) |
| 453 | + else: |
| 454 | + # Add explicit run stage args |
| 455 | + ni_barista_cmd += mx_sdk_benchmark.parse_prefixed_args("-Dnative-image.benchmark.extra-run-arg=", suite.context.bmSuiteArgs) |
| 456 | + if nivm_cmd_prefix: |
| 457 | + self._updateCommandOption(ni_barista_cmd, "--cmd-app-prefix", "-p", " ".join(nivm_cmd_prefix)) |
| 458 | + if nivm_app_options: |
| 459 | + self._updateCommandOption(ni_barista_cmd, "--app-args", "-a", " ".join(nivm_app_options)) |
| 460 | + ni_barista_cmd += [barista_bench_name] |
| 461 | + return ni_barista_cmd |
| 462 | + |
| 463 | + |
| 464 | +mx_benchmark.add_bm_suite(BaristaNativeImageBenchmarkSuite()) |
| 465 | + |
| 466 | + |
267 | 467 | class BaseDaCapoNativeImageBenchmarkSuite(): |
268 | 468 |
|
269 | 469 | '''`SetBuildInfo` method in DaCapo source reads from the file nested in daCapo jar. |
|
0 commit comments