Add basic thresholds for PackageGraphBenchmarks.swift (swiftlang#7462)

This records thresholds for two relatively stable metrics: total malloc
count and total number of syscalls. We don't run these benchmarks on CI
yet, but now it allows tracking regressions locally.

Author: MaxDesiatov

Benchmarks/Benchmarks/PackageGraphBenchmarks/PackageGraphBenchmarks.swift

@@ -1,14 +1,34 @@
 import Basics
 import Benchmark
+import Foundation
 import PackageModel
 import Workspace
 
 let benchmarks = {
+    let defaultMetrics: [BenchmarkMetric]
+    if let envVar = ProcessInfo.processInfo.environment["SWIFTPM_BENCHMARK_ALL_METRICS"],
+    envVar.lowercased() == "true" || envVar == "1" {
+        defaultMetrics = .all
+    } else {
+        defaultMetrics = [
+            .mallocCountTotal,
+            .syscalls,
+        ]
+    }
+
+    // Benchmarks computation of a resolved graph of modules for a package using `Workspace` as an entry point. It runs PubGrub to get
+    // resolved concrete versions of dependencies, assigning all modules and products to each other as corresponding dependencies
+    // with their build triples, but with the build plan not yet constructed. In this benchmark specifically we're loading `Package.swift`
+    // for SwiftPM itself.
     Benchmark(
-        "Package graph loading",
+        "SwiftPMWorkspaceModulesGraph",
         configuration: .init(
-            metrics: BenchmarkMetric.all,
-            maxDuration: .seconds(10)
+            metrics: defaultMetrics,
+            maxDuration: .seconds(10),
+            thresholds: [
+                .mallocCountTotal: .init(absolute: [.p90: 12000]),
+                .syscalls: .init(absolute: [.p90: 1600]),
+            ]
         )
     ) { benchmark in
         let path = try AbsolutePath(validating: #file).parentDirectory.parentDirectory.parentDirectory

Benchmarks/README.md

@@ -0,0 +1,30 @@
+# SwiftPM Benchmarks
+
+Benchmarks currently use [ordo-one/package-benchmark](https://github.com/ordo-one/package-benchmark) library for benchmarking.
+
+## How to Run
+
+To run the benchmarks in their default configuration, run this commend in the `Benchmarks` subdirectory of the SwiftPM repository clone (the directory in which this `README.md` file is contained):
+```
+swift package benchmark
+```
+
+To collect all benchmark metrics, set `SWIFTPM_BENCHMARK_ALL_METRICS` to a truthy value:
+
+```
+SWIFTPM_BENCHMARK_ALL_METRICS=true swift package benchmark
+```
+
+## Benchmark Thresholds
+
+`Benchmarks/Thresholds` subdirectory contains recorded allocation and syscall counts for macOS on Apple Silicon when built with Swift 5.10. To record new thresholds, run the following command:
+
+```
+swift package --allow-writing-to-package-directory benchmark --format metricP90AbsoluteThresholds --path Thresholds/
+```
+
+To verify that recorded thresholds do not exceeded given relative or absolute values (passed as `thresholds` arguments to each benchmark's configuration), run this command:
+
+```
+swift package benchmark baseline check --check-absolute-path Thresholds/
+```

Benchmarks/Thresholds/PackageGraphBenchmarks.PackageGraphLoading.p90.json

@@ -0,0 +1,4 @@
+{
+  "mallocCountTotal" : 9535,
+  "syscalls" : 1496
+}