From 16af31a39b201b9531cc36b8fbd7e761cc8a5617 Mon Sep 17 00:00:00 2001 From: Pavol Vaskovic Date: Sun, 17 Feb 2019 07:09:14 +0100 Subject: [PATCH 1/3] [benchmark] Remove HashQuadratic This was obsoleted by DictionaryCopy. --- benchmark/CMakeLists.txt | 1 - benchmark/single-source/HashQuadratic.swift | 37 --------------------- benchmark/utils/main.swift | 2 -- 3 files changed, 40 deletions(-) delete mode 100644 benchmark/single-source/HashQuadratic.swift diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index 21da01d575207..8124d4ccf2a99 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -89,7 +89,6 @@ set(SWIFT_BENCH_MODULES single-source/FloatingPointPrinting single-source/Hanoi single-source/Hash - single-source/HashQuadratic single-source/Histogram single-source/InsertCharacter single-source/Integrate diff --git a/benchmark/single-source/HashQuadratic.swift b/benchmark/single-source/HashQuadratic.swift deleted file mode 100644 index 39665547c0b25..0000000000000 --- a/benchmark/single-source/HashQuadratic.swift +++ /dev/null @@ -1,37 +0,0 @@ -//===--- HashQuadratic.swift ----------------------------------------------===// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// - -import TestsUtils - -public let HashQuadratic = BenchmarkInfo( - name: "HashQuadratic", - runFunction: run_HashQuadratic, - tags: [.unstable, .api, .Dictionary]) - -let size = 3_000_000 - -@inline(never) -public func run_HashQuadratic(_ N: Int) { - for _ in 1...N { - var dict1: [Int: Int] = [:] - for i in 0.. Date: Sat, 16 Feb 2019 23:23:28 +0100 Subject: [PATCH 2/3] [benchmark] Dict.[CopyKeyValue, FilterAllMatch] Split the composite tests from `DictionatyCopy` and `DictionaryFilter` into individual benchmarks by dictionary size. Lowered the workloads to run faster (more stable results). --- benchmark/single-source/DictionaryCopy.swift | 88 +++++++++----------- 1 file changed, 38 insertions(+), 50 deletions(-) diff --git a/benchmark/single-source/DictionaryCopy.swift b/benchmark/single-source/DictionaryCopy.swift index c035f09cbfbc2..7a03705d70646 100644 --- a/benchmark/single-source/DictionaryCopy.swift +++ b/benchmark/single-source/DictionaryCopy.swift @@ -17,66 +17,54 @@ import TestsUtils -public let DictionaryCopy = [ - BenchmarkInfo(name: "DictionaryCopy", runFunction: run_DictionaryCopy, - tags: [.validation, .api, .Dictionary]), - BenchmarkInfo(name: "DictionaryFilter", runFunction: run_DictionaryFilter, - tags: [.validation, .api, .Dictionary]), -] +let t: [BenchmarkCategory] = [.validation, .api, .Dictionary] -@inline(never) -public func testCopy(_ size: Int) { - var dict1 = [Int: Int](minimumCapacity: size) +// We run the test at a spread of sizes between 1*x and 2*x, because the +// quadratic behavior only happens at certain load factors. - // Fill dictionary - for i in 1...size { - dict1[i] = 2 * i - } - CheckResults(dict1.count == size) +public let DictionaryCopy = [ + BenchmarkInfo(name:"Dict.CopyKeyValue.16k", + runFunction: copyKeyValue, tags: t, setUpFunction: { dict(16_000) }), + BenchmarkInfo(name:"Dict.CopyKeyValue.20k", + runFunction: copyKeyValue, tags: t, setUpFunction: { dict(20_000) }), + BenchmarkInfo(name:"Dict.CopyKeyValue.24k", + runFunction: copyKeyValue, tags: t, setUpFunction: { dict(24_000) }), + BenchmarkInfo(name:"Dict.CopyKeyValue.28k", + runFunction: copyKeyValue, tags: t, setUpFunction: { dict(28_000) }), - var dict2 = [Int: Int]() - for (key, value) in dict1 { - dict2[key] = value - } - CheckResults(dict2.count == size) -} + BenchmarkInfo(name:"Dict.FilterAllMatch.16k", + runFunction: filterAllMatch, tags: t, setUpFunction: { dict(16_000) }), + BenchmarkInfo(name:"Dict.FilterAllMatch.20k", + runFunction: filterAllMatch, tags: t, setUpFunction: { dict(20_000) }), + BenchmarkInfo(name:"Dict.FilterAllMatch.24k", + runFunction: filterAllMatch, tags: t, setUpFunction: { dict(24_000) }), + BenchmarkInfo(name:"Dict.FilterAllMatch.28k", + runFunction: filterAllMatch, tags: t, setUpFunction: { dict(28_000) }), +] -@inline(never) -public func run_DictionaryCopy(_ N: Int) { - for _ in 0 ..< N { - // We run the test at a spread of sizes between 1*x and 2*x, because the - // quadratic behavior only happens at certain load factors. - testCopy(100_000) - testCopy(125_000) - testCopy(150_000) - testCopy(175_000) - } +var dict: [Int: Int]? + +func dict(_ size: Int) { + dict = Dictionary(uniqueKeysWithValues: zip(1...size, 1...size)) } @inline(never) -public func testFilter(_ size: Int) { - var dict1 = [Int: Int](minimumCapacity: size) - - // Fill dictionary - for i in 1...size { - dict1[i] = 2 * i +func copyKeyValue(N: Int) { + for _ in 1...N { + var copy = [Int: Int]() + for (key, value) in dict! { + copy[key] = value + } + CheckResults(copy.count == dict!.count) } - CheckResults(dict1.count == size) - - // Filter with a predicate returning true is essentially the same loop as the - // one in testCopy above. - let dict2 = dict1.filter { _ in true } - CheckResults(dict2.count == size) } +// Filter with a predicate returning true is essentially the same loop as the +// one in copyKeyValue above. @inline(never) -public func run_DictionaryFilter(_ N: Int) { - for _ in 0 ..< N { - // We run the test at a spread of sizes between 1*x and 2*x, because the - // quadratic behavior only happens at certain load factors. - testFilter(100_000) - testFilter(125_000) - testFilter(150_000) - testFilter(175_000) +func filterAllMatch(N: Int) { + for _ in 1...N { + let copy = dict!.filter { _ in true } + CheckResults(copy.count == dict!.count) } } From e13e90ba77c38662c826d3579ef75e71e2cd9d10 Mon Sep 17 00:00:00 2001 From: Pavol Vaskovic Date: Sun, 17 Feb 2019 19:04:51 +0100 Subject: [PATCH 3/3] [benchmark] Adjust lit tests for Benchmark_O Change the test to work after removal of `HashQuadratic`. --- test/benchmark/Benchmark_O.test.md | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/test/benchmark/Benchmark_O.test.md b/test/benchmark/Benchmark_O.test.md index 752c0c2b6e596..20b2754aff257 100644 --- a/test/benchmark/Benchmark_O.test.md +++ b/test/benchmark/Benchmark_O.test.md @@ -14,9 +14,9 @@ as a verification of this public API to prevent its accidental breakage. [BD]: https://github.com/apple/swift/blob/master/benchmark/scripts/Benchmark_Driver [Testing]: https://github.com/apple/swift/blob/master/docs/Testing.md -Note: Following tests use *HashQuadratic* as an example of a benchmark that is -excluded from the default "pre-commit" list because it is marked `unstable` and -the default skip-tags (`unstable,skip`) will exclude it. The *Ackermann* and +Note: Following tests use *Existential.* as an example of a benchmarks that are +excluded from the default "pre-commit" list because they are marked `skip` and +the default skip-tags (`unstable,skip`) will exclude them. The *Ackermann* and *AngryPhonebook* are alphabetically the first two benchmarks in the test suite (used to verify running by index). If these assumptions change, the test must be adapted. @@ -27,7 +27,7 @@ RUN: %Benchmark_O --list | %FileCheck %s \ RUN: --check-prefix LISTPRECOMMIT \ RUN: --check-prefix LISTTAGS LISTPRECOMMIT: #,Test,[Tags] -LISTPRECOMMIT-NOT: HashQuadratic +LISTPRECOMMIT-NOT: Existential. LISTPRECOMMIT: {{[0-9]+}},AngryPhonebook LISTTAGS-SAME: ,[ LISTTAGS-NOT: TestsUtils.BenchmarkCategory. @@ -35,14 +35,14 @@ LISTTAGS-SAME: String, api, validation LISTTAGS-SAME: ] ```` -Verify HashQuadratic is listed when skip-tags are explicitly empty and that it -is marked unstable: +Verify `Existential.` benchmarks are listed when skip-tags are explicitly empty +and that they are marked `skip`: ```` RUN: %Benchmark_O --list --skip-tags= | %FileCheck %s --check-prefix LISTALL LISTALL: AngryPhonebook -LISTALL: HashQuadratic -LISTALL-SAME: unstable +LISTALL: Existential. +LISTALL-SAME: skip ```` ## Benchmark Selection @@ -54,8 +54,9 @@ It provides us with ability to do a "dry run". Run benchmark by name (even if its tags match the skip-tags) or test number: ```` -RUN: %Benchmark_O HashQuadratic --list | %FileCheck %s --check-prefix NAMEDSKIP -NAMEDSKIP: HashQuadratic +RUN: %Benchmark_O Existential.Mutating.Ref1 --list \ +RUN: | %FileCheck %s --check-prefix NAMEDSKIP +NAMEDSKIP: Existential.Mutating.Ref1 RUN: %Benchmark_O 1 --list | %FileCheck %s --check-prefix RUNBYNUMBER RUNBYNUMBER: Ackermann