From 7c220ffb3882aa12b786f3e216790fb393a9c9f6 Mon Sep 17 00:00:00 2001 From: MrLotU Date: Wed, 4 Aug 2021 10:49:05 +0200 Subject: [PATCH 1/7] Move to GH actions --- .github/workflows/test.yml | 33 +++++++++++++++++++++++++++++++++ circle.yml | 34 ---------------------------------- 2 files changed, 33 insertions(+), 34 deletions(-) create mode 100644 .github/workflows/test.yml delete mode 100644 circle.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..078f116 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,33 @@ +name: tests +on: { pull_request: {} } + +jobs: + linux: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + docker: ['swift:5.2-focal', 'swift:5.3-focal', 'swift:5.4-focal'] + container: ${{ matrix.docker }} + steps: + - name: Check out SwiftPrometheus + uses: actions/checkout@v2 + - name: Run tests with Thread Sanitizer + timeout-minutes: 30 + run: swift test --enable-test-discovery --sanitize=thread + macos: + runs-on: macos-latest + strategy: + fail-fast: false + matrix: + toolchain: ['latest', 'latest-stable'] + steps: + - name: Select toolchain + uses: maxim-lobanov/setup-xcode@v1.2.1 + with: + xcode-version: ${{ matrix.toolchain }} + - name: Check out SwiftPrometheus + uses: actions/checkout@v2 + - name: Run tests with Thread Sanitizer + timeout-minutes: 30 + run: swift test --enable-test-discovery --sanitize=thread diff --git a/circle.yml b/circle.yml deleted file mode 100644 index 76417f7..0000000 --- a/circle.yml +++ /dev/null @@ -1,34 +0,0 @@ -version: 2 - -jobs: - swift5: - docker: - - image: swift:5.2 - steps: - - checkout - - run: apt-get update; apt-get install -y libssl-dev zlib1g-dev - - run: swift build - - run: swift test - bionic: - docker: - - image: vapor/swift:5.2-bionic - steps: - - checkout - - run: apt-get update; apt-get install -y libssl-dev zlib1g-dev - - run: swift build - - run: swift test - bionic-release: - docker: - - image: vapor/swift:5.2-bionic - steps: - - checkout - - run: apt-get update; apt-get install -y libssl-dev zlib1g-dev - - run: swift build -c release - -workflows: - version: 2 - tests: - jobs: - - bionic - - bionic-release - - swift5 From db0dad805eb48a6ebf2e143a8235a07a817a162b Mon Sep 17 00:00:00 2001 From: MrLotU Date: Tue, 24 Aug 2021 10:37:16 +0200 Subject: [PATCH 2/7] Make time based tests less flakey --- Tests/SwiftPrometheusTests/GaugeTests.swift | 2 +- Tests/SwiftPrometheusTests/SummaryTests.swift | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Tests/SwiftPrometheusTests/GaugeTests.swift b/Tests/SwiftPrometheusTests/GaugeTests.swift index a0fb8fd..ab3cbcc 100644 --- a/Tests/SwiftPrometheusTests/GaugeTests.swift +++ b/Tests/SwiftPrometheusTests/GaugeTests.swift @@ -55,7 +55,7 @@ final class GaugeTests: XCTestCase { func testGaugeTime() { let gauge = prom.createGauge(forType: Double.self, named: "my_gauge") - let delay = 0.05 + let delay = 0.051 gauge.time { Thread.sleep(forTimeInterval: delay) } diff --git a/Tests/SwiftPrometheusTests/SummaryTests.swift b/Tests/SwiftPrometheusTests/SummaryTests.swift index d835791..5e868a0 100644 --- a/Tests/SwiftPrometheusTests/SummaryTests.swift +++ b/Tests/SwiftPrometheusTests/SummaryTests.swift @@ -99,7 +99,7 @@ final class SummaryTests: XCTestCase { func testSummaryTime() { let summary = prom.createSummary(forType: Double.self, named: "my_summary", helpText: "Summary for testing", quantiles: [0.5, 0.9, 0.99], labels: BaseSummaryLabels.self) - let delay = 0.05 + let delay = 0.051 summary.time { Thread.sleep(forTimeInterval: delay) } From e139f9e827e9ee6cb250033f9624a27a3bd9aa46 Mon Sep 17 00:00:00 2001 From: MrLotU Date: Tue, 24 Aug 2021 10:43:05 +0200 Subject: [PATCH 3/7] Add debug output to failing tests --- Tests/SwiftPrometheusTests/GaugeTests.swift | 5 +++-- Tests/SwiftPrometheusTests/HistogramTests.swift | 5 +++-- Tests/SwiftPrometheusTests/SummaryTests.swift | 5 +++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/Tests/SwiftPrometheusTests/GaugeTests.swift b/Tests/SwiftPrometheusTests/GaugeTests.swift index ab3cbcc..b04a5c1 100644 --- a/Tests/SwiftPrometheusTests/GaugeTests.swift +++ b/Tests/SwiftPrometheusTests/GaugeTests.swift @@ -60,10 +60,11 @@ final class GaugeTests: XCTestCase { Thread.sleep(forTimeInterval: delay) } // Using starts(with:) here since the exact subseconds might differ per-test. - XCTAssert(gauge.collect().starts(with: """ + let output = gauge.collect() + XCTAssert(output.starts(with: """ # TYPE my_gauge gauge my_gauge 0.05 - """)) + """), output) } func testGaugeStandalone() { diff --git a/Tests/SwiftPrometheusTests/HistogramTests.swift b/Tests/SwiftPrometheusTests/HistogramTests.swift index 4d7ab3c..1c16a1f 100644 --- a/Tests/SwiftPrometheusTests/HistogramTests.swift +++ b/Tests/SwiftPrometheusTests/HistogramTests.swift @@ -86,7 +86,8 @@ final class HistogramTests: XCTestCase { Thread.sleep(forTimeInterval: delay) } // Using starts(with:) here since the exact subseconds might differ per-test. - XCTAssert(histogram.collect().starts(with: """ + let output = histogram.collect() + XCTAssert(output.starts(with: """ # TYPE my_histogram histogram my_histogram_bucket{le="0.005"} 0.0 my_histogram_bucket{le="0.01"} 0.0 @@ -102,7 +103,7 @@ final class HistogramTests: XCTestCase { my_histogram_bucket{le="+Inf"} 1.0 my_histogram_count 1.0 my_histogram_sum 0.05 - """)) + """), output) } func testHistogramStandalone() { diff --git a/Tests/SwiftPrometheusTests/SummaryTests.swift b/Tests/SwiftPrometheusTests/SummaryTests.swift index b9b6051..a4a6665 100644 --- a/Tests/SwiftPrometheusTests/SummaryTests.swift +++ b/Tests/SwiftPrometheusTests/SummaryTests.swift @@ -114,8 +114,9 @@ final class SummaryTests: XCTestCase { #"my_summary_count{myValue="*"} 1.0"#, #"my_summary_sum{myValue="*"} 0.05"# ] - let sections = summary.collect().split(separator: "\n").map(String.init).enumerated().map { i, s in s.starts(with: lines[i]) } - XCTAssert(sections.filter { !$0 }.isEmpty) + let output = summary.collect() + let sections = output.split(separator: "\n").map(String.init).enumerated().map { i, s in s.starts(with: lines[i]) } + XCTAssert(sections.filter { !$0 }.isEmpty, output) } func testSummaryStandalone() { From 89d36a7b126b748379c2e4049744ce2bed68a680 Mon Sep 17 00:00:00 2001 From: MrLotU Date: Tue, 24 Aug 2021 10:52:49 +0200 Subject: [PATCH 4/7] Don't run timing based tests on MacOS --- Tests/SwiftPrometheusTests/GaugeTests.swift | 9 +++++---- Tests/SwiftPrometheusTests/HistogramTests.swift | 9 +++++---- Tests/SwiftPrometheusTests/SummaryTests.swift | 11 ++++++----- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/Tests/SwiftPrometheusTests/GaugeTests.swift b/Tests/SwiftPrometheusTests/GaugeTests.swift index b04a5c1..b02a92a 100644 --- a/Tests/SwiftPrometheusTests/GaugeTests.swift +++ b/Tests/SwiftPrometheusTests/GaugeTests.swift @@ -53,19 +53,20 @@ final class GaugeTests: XCTestCase { """) } + #if os(Linux) func testGaugeTime() { let gauge = prom.createGauge(forType: Double.self, named: "my_gauge") - let delay = 0.051 + let delay = 0.05 gauge.time { Thread.sleep(forTimeInterval: delay) } // Using starts(with:) here since the exact subseconds might differ per-test. - let output = gauge.collect() - XCTAssert(output.starts(with: """ + XCTAssert(gauge.collect().starts(with: """ # TYPE my_gauge gauge my_gauge 0.05 - """), output) + """)) } + #endif func testGaugeStandalone() { let gauge = prom.createGauge(forType: Int.self, named: "my_gauge", helpText: "Gauge for testing", initialValue: 10, withLabelType: BaseLabels.self) diff --git a/Tests/SwiftPrometheusTests/HistogramTests.swift b/Tests/SwiftPrometheusTests/HistogramTests.swift index 1c16a1f..321b57a 100644 --- a/Tests/SwiftPrometheusTests/HistogramTests.swift +++ b/Tests/SwiftPrometheusTests/HistogramTests.swift @@ -78,7 +78,8 @@ final class HistogramTests: XCTestCase { my_histogram_sum{myValue="labels"} 3.0\n """) } - + + #if os(Linux) func testHistogramTime() { let histogram = prom.createHistogram(forType: Double.self, named: "my_histogram") let delay = 0.05 @@ -86,8 +87,7 @@ final class HistogramTests: XCTestCase { Thread.sleep(forTimeInterval: delay) } // Using starts(with:) here since the exact subseconds might differ per-test. - let output = histogram.collect() - XCTAssert(output.starts(with: """ + XCTAssert(histogram.collect().starts(with: """ # TYPE my_histogram histogram my_histogram_bucket{le="0.005"} 0.0 my_histogram_bucket{le="0.01"} 0.0 @@ -103,8 +103,9 @@ final class HistogramTests: XCTestCase { my_histogram_bucket{le="+Inf"} 1.0 my_histogram_count 1.0 my_histogram_sum 0.05 - """), output) + """)) } + #endif func testHistogramStandalone() { let histogram = prom.createHistogram(forType: Double.self, named: "my_histogram", helpText: "Histogram for testing", buckets: [0.5, 1, 2, 3, 5, Double.greatestFiniteMagnitude], labels: BaseHistogramLabels.self) diff --git a/Tests/SwiftPrometheusTests/SummaryTests.swift b/Tests/SwiftPrometheusTests/SummaryTests.swift index a4a6665..20e2969 100644 --- a/Tests/SwiftPrometheusTests/SummaryTests.swift +++ b/Tests/SwiftPrometheusTests/SummaryTests.swift @@ -96,10 +96,11 @@ final class SummaryTests: XCTestCase { my_summary_sum 10010.0\n """) } - + + #if os(Linux) func testSummaryTime() { let summary = prom.createSummary(forType: Double.self, named: "my_summary", helpText: "Summary for testing", quantiles: [0.5, 0.9, 0.99], labels: BaseSummaryLabels.self) - let delay = 0.051 + let delay = 0.05 summary.time { Thread.sleep(forTimeInterval: delay) } @@ -114,10 +115,10 @@ final class SummaryTests: XCTestCase { #"my_summary_count{myValue="*"} 1.0"#, #"my_summary_sum{myValue="*"} 0.05"# ] - let output = summary.collect() - let sections = output.split(separator: "\n").map(String.init).enumerated().map { i, s in s.starts(with: lines[i]) } - XCTAssert(sections.filter { !$0 }.isEmpty, output) + let sections = summary.collect().split(separator: "\n").map(String.init).enumerated().map { i, s in s.starts(with: lines[i]) } + XCTAssert(sections.filter { !$0 }.isEmpty) } + #endif func testSummaryStandalone() { let summary = prom.createSummary(forType: Double.self, named: "my_summary", helpText: "Summary for testing", quantiles: [0.5, 0.9, 0.99], labels: BaseSummaryLabels.self) From 89f94d71db96654010bdddc694cba7cf3e9091c4 Mon Sep 17 00:00:00 2001 From: MrLotU Date: Fri, 3 Jun 2022 14:53:01 +0200 Subject: [PATCH 5/7] chore: Work around time-flakyness --- .github/workflows/test.yml | 14 ++++++++++++-- Package.resolved | 8 ++++---- Tests/SwiftPrometheusTests/GaugeTests.swift | 6 ++---- Tests/SwiftPrometheusTests/HistogramTests.swift | 6 ++---- Tests/SwiftPrometheusTests/SummaryTests.swift | 10 ++++------ .../SwiftPrometheusTests.swift | 6 ++++++ 6 files changed, 30 insertions(+), 20 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 078f116..76f45b4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,13 +7,21 @@ jobs: strategy: fail-fast: false matrix: - docker: ['swift:5.2-focal', 'swift:5.3-focal', 'swift:5.4-focal'] - container: ${{ matrix.docker }} + swiftver: + - swift:5.4 + - swift:5.5 + - swift:5.6 + - swiftlang/swift:nightly-main + swiftos: + - focal + container: ${{ format('{0}-{1}', matrix.swiftver, matrix.swiftos) }} steps: - name: Check out SwiftPrometheus uses: actions/checkout@v2 - name: Run tests with Thread Sanitizer timeout-minutes: 30 + env: + CI_RUN: 'TRUE' run: swift test --enable-test-discovery --sanitize=thread macos: runs-on: macos-latest @@ -30,4 +38,6 @@ jobs: uses: actions/checkout@v2 - name: Run tests with Thread Sanitizer timeout-minutes: 30 + env: + CI_RUN: 'TRUE' run: swift test --enable-test-discovery --sanitize=thread diff --git a/Package.resolved b/Package.resolved index fc1fd46..b9b77eb 100644 --- a/Package.resolved +++ b/Package.resolved @@ -6,8 +6,8 @@ "repositoryURL": "https://github.com/apple/swift-metrics.git", "state": { "branch": null, - "revision": "3edd2f57afc4e68e23c3e4956bc8b65ca6b5b2ff", - "version": "2.2.0" + "revision": "1c1408bf8fc21be93713e897d2badf500ea38419", + "version": "2.3.1" } }, { @@ -15,8 +15,8 @@ "repositoryURL": "https://github.com/apple/swift-nio.git", "state": { "branch": null, - "revision": "9201908b54578aa33f1d1826a5a680aca8991843", - "version": "2.8.0" + "revision": "124119f0bb12384cef35aa041d7c3a686108722d", + "version": "2.40.0" } } ] diff --git a/Tests/SwiftPrometheusTests/GaugeTests.swift b/Tests/SwiftPrometheusTests/GaugeTests.swift index b02a92a..6619119 100644 --- a/Tests/SwiftPrometheusTests/GaugeTests.swift +++ b/Tests/SwiftPrometheusTests/GaugeTests.swift @@ -53,7 +53,6 @@ final class GaugeTests: XCTestCase { """) } - #if os(Linux) func testGaugeTime() { let gauge = prom.createGauge(forType: Double.self, named: "my_gauge") let delay = 0.05 @@ -63,11 +62,10 @@ final class GaugeTests: XCTestCase { // Using starts(with:) here since the exact subseconds might differ per-test. XCTAssert(gauge.collect().starts(with: """ # TYPE my_gauge gauge - my_gauge 0.05 + my_gauge \(isCITestRun ? "" : "0.05") """)) } - #endif - + func testGaugeStandalone() { let gauge = prom.createGauge(forType: Int.self, named: "my_gauge", helpText: "Gauge for testing", initialValue: 10, withLabelType: BaseLabels.self) XCTAssertEqual(gauge.get(), 10) diff --git a/Tests/SwiftPrometheusTests/HistogramTests.swift b/Tests/SwiftPrometheusTests/HistogramTests.swift index ca537b1..dc5b16a 100644 --- a/Tests/SwiftPrometheusTests/HistogramTests.swift +++ b/Tests/SwiftPrometheusTests/HistogramTests.swift @@ -113,7 +113,6 @@ final class HistogramTests: XCTestCase { """) } - #if os(Linux) func testHistogramTime() { let histogram = prom.createHistogram(forType: Double.self, named: "my_histogram") let delay = 0.05 @@ -136,11 +135,10 @@ final class HistogramTests: XCTestCase { my_histogram_bucket{le="10.0"} 1.0 my_histogram_bucket{le="+Inf"} 1.0 my_histogram_count 1.0 - my_histogram_sum 0.05 + my_histogram_sum \(isCITestRun ? "" : "0.05") """)) } - #endif - + func testHistogramStandalone() { let histogram = prom.createHistogram(forType: Double.self, named: "my_histogram", helpText: "Histogram for testing", buckets: [0.5, 1, 2, 3, 5, Double.greatestFiniteMagnitude], labels: BaseHistogramLabels.self) let histogramTwo = prom.createHistogram(forType: Double.self, named: "my_histogram", helpText: "Histogram for testing", buckets: [0.5, 1, 2, 3, 5, Double.greatestFiniteMagnitude], labels: BaseHistogramLabels.self) diff --git a/Tests/SwiftPrometheusTests/SummaryTests.swift b/Tests/SwiftPrometheusTests/SummaryTests.swift index d43c29b..61e6e0e 100644 --- a/Tests/SwiftPrometheusTests/SummaryTests.swift +++ b/Tests/SwiftPrometheusTests/SummaryTests.swift @@ -131,7 +131,6 @@ final class SummaryTests: XCTestCase { """) } - #if os(Linux) func testSummaryTime() { let summary = prom.createSummary(forType: Double.self, named: "my_summary", helpText: "Summary for testing", quantiles: [0.5, 0.9, 0.99], labels: BaseSummaryLabels.self) let delay = 0.05 @@ -143,16 +142,15 @@ final class SummaryTests: XCTestCase { let lines = [ "# HELP my_summary Summary for testing", "# TYPE my_summary summary", - #"my_summary{quantile="0.5", myValue="*"} 0.05"#, - #"my_summary{quantile="0.9", myValue="*"} 0.05"#, - #"my_summary{quantile="0.99", myValue="*"} 0.05"#, + #"my_summary{quantile="0.5", myValue="*"} \#(isCITestRun ? "" : "0.05")"#, + #"my_summary{quantile="0.9", myValue="*"} \#(isCITestRun ? "" : "0.05")"#, + #"my_summary{quantile="0.99", myValue="*"} \#(isCITestRun ? "" : "0.05")"#, #"my_summary_count{myValue="*"} 1.0"#, - #"my_summary_sum{myValue="*"} 0.05"# + #"my_summary_sum{myValue="*"} \#(isCITestRun ? "" : "0.05")"# ] let sections = summary.collect().split(separator: "\n").map(String.init).enumerated().map { i, s in s.starts(with: lines[i]) } XCTAssert(sections.filter { !$0 }.isEmpty) } - #endif func testSummaryStandalone() { let summary = prom.createSummary(forType: Double.self, named: "my_summary", helpText: "Summary for testing", quantiles: [0.5, 0.9, 0.99], labels: BaseSummaryLabels.self) diff --git a/Tests/SwiftPrometheusTests/SwiftPrometheusTests.swift b/Tests/SwiftPrometheusTests/SwiftPrometheusTests.swift index e0e662e..f2f124e 100644 --- a/Tests/SwiftPrometheusTests/SwiftPrometheusTests.swift +++ b/Tests/SwiftPrometheusTests/SwiftPrometheusTests.swift @@ -1,6 +1,12 @@ import XCTest @testable import Prometheus +var isCITestRun: Bool { + return ProcessInfo.processInfo.environment.contains { k, v in + return k == "CI_RUN" && v == "TRUE" + } +} + final class SwiftPrometheusTests: XCTestCase { struct BaseLabels: MetricLabels { From 2652588dba74051046418d96aae6db5b7212562c Mon Sep 17 00:00:00 2001 From: MrLotU Date: Fri, 3 Jun 2022 14:59:01 +0200 Subject: [PATCH 6/7] chore: Re add swift 5.2 and 5.3 --- .github/workflows/test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 76f45b4..c89850a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,6 +8,8 @@ jobs: fail-fast: false matrix: swiftver: + - swift:5.2 + - swift:5.3 - swift:5.4 - swift:5.5 - swift:5.6 From 6e6a4ff8faa5c0fdf306d25ef5408abe75c241a6 Mon Sep 17 00:00:00 2001 From: MrLotU Date: Fri, 3 Jun 2022 15:02:47 +0200 Subject: [PATCH 7/7] chore: Update the histogram test to allow for even more varying CI times --- Tests/SwiftPrometheusTests/HistogramTests.swift | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/Tests/SwiftPrometheusTests/HistogramTests.swift b/Tests/SwiftPrometheusTests/HistogramTests.swift index dc5b16a..dd82ebe 100644 --- a/Tests/SwiftPrometheusTests/HistogramTests.swift +++ b/Tests/SwiftPrometheusTests/HistogramTests.swift @@ -119,23 +119,15 @@ final class HistogramTests: XCTestCase { histogram.time { Thread.sleep(forTimeInterval: delay) } - // Using starts(with:) here since the exact subseconds might differ per-test. - XCTAssert(histogram.collect().starts(with: """ - # TYPE my_histogram histogram - my_histogram_bucket{le="0.005"} 0.0 - my_histogram_bucket{le="0.01"} 0.0 - my_histogram_bucket{le="0.025"} 0.0 - my_histogram_bucket{le="0.05"} 0.0 - my_histogram_bucket{le="0.1"} 1.0 - my_histogram_bucket{le="0.25"} 1.0 - my_histogram_bucket{le="0.5"} 1.0 + // Using `contains` here since the exact subseconds might differ per-test, and CI runners can vary even more. + XCTAssert(histogram.collect().contains(""" my_histogram_bucket{le="1.0"} 1.0 my_histogram_bucket{le="2.5"} 1.0 my_histogram_bucket{le="5.0"} 1.0 my_histogram_bucket{le="10.0"} 1.0 my_histogram_bucket{le="+Inf"} 1.0 my_histogram_count 1.0 - my_histogram_sum \(isCITestRun ? "" : "0.05") + my_histogram_sum """)) }