Skip to content

Commit 989690f

Browse files
estroztimflannagan
andauthored
chore(e2e): naively parallelize CI jobs by chunking alphabetically (#2520)
* chore(e2e): naively parallelize CI jobs by chunking alphabetically Signed-off-by: Eric Stroczynski <[email protected]> * build word trie from specs, add unit tests Signed-off-by: Eric Stroczynski <[email protected]> * .github: Aggregate e2e matrix jobs into a single status Signed-off-by: timflannagan <[email protected]> Co-authored-by: timflannagan <[email protected]>
1 parent 97bd070 commit 989690f

File tree

5 files changed

+410
-4
lines changed

5 files changed

+410
-4
lines changed

.github/workflows/e2e-tests.yml

+20-3
Original file line numberDiff line numberDiff line change
@@ -8,17 +8,34 @@ on:
88
pull_request:
99
workflow_dispatch:
1010
jobs:
11-
e2e-tests:
11+
e2e:
12+
strategy:
13+
fail-fast: false
14+
matrix:
15+
parallel-id: [0, 1, 2, 3]
1216
runs-on: ubuntu-latest
1317
steps:
1418
- uses: actions/checkout@v1
1519
- uses: actions/setup-go@v2
1620
with:
1721
go-version: '~1.17'
18-
- run: make e2e-local E2E_NODES=2 ARTIFACTS_DIR=./artifacts/
22+
- run: make e2e-local E2E_TEST_CHUNK=${{ matrix.parallel-id }} E2E_TEST_NUM_CHUNKS=${{ strategy.job-total }} E2E_NODES=2 ARTIFACTS_DIR=./artifacts/
1923
- name: Archive Test Artifacts # test results, failed or not, are always uploaded.
2024
if: ${{ always() }}
2125
uses: actions/upload-artifact@v2
2226
with:
23-
name: e2e-test-output-${{(github.event.pull_request.head.sha||github.sha)}}-${{ github.run_id }}
27+
name: e2e-test-output-${{ (github.event.pull_request.head.sha || github.sha) }}-${{ github.run_id }}-${{ matrix.parallel-id }}
2428
path: ${{ github.workspace }}/bin/artifacts/*
29+
# TODO: create job to combine test artifacts using code in https://github.com/operator-framework/operator-lifecycle-manager/pull/1476
30+
e2e-tests:
31+
if: ${{ always() }}
32+
runs-on: ubuntu-latest
33+
needs: e2e
34+
steps:
35+
- run: |
36+
echo "Matrix result: ${{ needs.e2e.result }}"
37+
- name: check individual matrix results
38+
if: ${{ needs.e2e.result == 'failure' }}
39+
run: |
40+
echo 'Failure: at least one e2e matrix job has failed'
41+
exit 1

Makefile

+8-1
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ all: test build
4343
test: clean cover.out
4444

4545
unit: kubebuilder
46-
KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS) go test $(MOD_FLAGS) $(SPECIFIC_UNIT_TEST) -tags "json1" -race -count=1 ./pkg/...
46+
KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS) go test $(MOD_FLAGS) $(SPECIFIC_UNIT_TEST) -tags "json1" -race -count=1 ./pkg/... ./test/e2e/split/...
4747

4848
# Ensure kubebuilder is installed before continuing
4949
KUBEBUILDER_ASSETS_ERR := not detected in $(KUBEBUILDER_ASSETS), to override the assets path set the KUBEBUILDER_ASSETS environment variable, for install instructions see https://book.kubebuilder.io/quick-start.html
@@ -126,6 +126,13 @@ setup-bare: clean e2e.namespace
126126
E2E_NODES ?= 1
127127
E2E_FLAKE_ATTEMPTS ?= 1
128128
E2E_TIMEOUT ?= 90m
129+
# Optionally run an individual chunk of e2e test specs.
130+
# Do not use this from the CLI; this is intended to be used by CI only.
131+
E2E_TEST_CHUNK ?= all
132+
E2E_TEST_NUM_CHUNKS ?= 4
133+
ifneq (all,$(E2E_TEST_CHUNK))
134+
TEST := $(shell go run ./test/e2e/split/... -chunks $(E2E_TEST_NUM_CHUNKS) -print-chunk $(E2E_TEST_CHUNK) ./test/e2e)
135+
endif
129136
E2E_OPTS ?= $(if $(E2E_SEED),-seed '$(E2E_SEED)') $(if $(TEST),-focus '$(TEST)') -flakeAttempts $(E2E_FLAKE_ATTEMPTS) -nodes $(E2E_NODES) -timeout $(E2E_TIMEOUT) -v -randomizeSuites -race -trace -progress
130137
E2E_INSTALL_NS ?= operator-lifecycle-manager
131138
E2E_TEST_NS ?= operators

test/e2e/split/integration_test.sh

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#!/usr/bin/env bash
2+
3+
function get_total_specs() {
4+
go run github.com/onsi/ginkgo/ginkgo -noColor -dryRun -v -seed 1 "$@" ./test/e2e | grep -Po "Ran \K([0-9]+)(?= of .+ Specs in .+ seconds)"
5+
}
6+
7+
unfocused_specs=$(get_total_specs)
8+
regexp=$(go run ./test/e2e/split/... -chunks 1 -print-chunk 0 ./test/e2e)
9+
focused_specs=$(get_total_specs -focus "$regexp")
10+
11+
if ! [ $unfocused_specs -eq $focused_specs ]; then
12+
echo "expected number of unfocused specs $unfocused_specs to equal focus specs $focused_specs"
13+
exit 1
14+
fi

test/e2e/split/main.go

+227
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,227 @@
1+
package main
2+
3+
import (
4+
"flag"
5+
"fmt"
6+
"io"
7+
"io/ioutil"
8+
"log"
9+
"math"
10+
"os"
11+
"path/filepath"
12+
"regexp"
13+
"sort"
14+
"strings"
15+
)
16+
17+
// TODO: configurable log verbosity.
18+
19+
type options struct {
20+
numChunks int
21+
printChunk int
22+
printDebug bool
23+
writer io.Writer
24+
}
25+
26+
func main() {
27+
opts := options{
28+
writer: os.Stdout,
29+
}
30+
flag.IntVar(&opts.numChunks, "chunks", 1, "Number of chunks to create focus regexps for")
31+
flag.IntVar(&opts.printChunk, "print-chunk", 0, "Chunk to print a regexp for")
32+
flag.BoolVar(&opts.printDebug, "print-debug", false, "Print all spec prefixes in non-regexp format. Use for debugging")
33+
flag.Parse()
34+
35+
if opts.printChunk >= opts.numChunks {
36+
exitIfErr(fmt.Errorf("the chunk to print (%d) must be a smaller number than the number of chunks (%d)", opts.printChunk, opts.numChunks))
37+
}
38+
39+
dir := flag.Arg(0)
40+
if dir == "" {
41+
exitIfErr(fmt.Errorf("test directory required as the argument"))
42+
}
43+
44+
// Clean dir.
45+
var err error
46+
dir, err = filepath.Abs(dir)
47+
exitIfErr(err)
48+
wd, err := os.Getwd()
49+
exitIfErr(err)
50+
dir, err = filepath.Rel(wd, dir)
51+
exitIfErr(err)
52+
53+
exitIfErr(opts.run(dir))
54+
}
55+
56+
func exitIfErr(err error) {
57+
if err != nil {
58+
log.Fatal(err)
59+
}
60+
}
61+
62+
func (opts options) run(dir string) error {
63+
describes, err := findDescribes(dir)
64+
if err != nil {
65+
return err
66+
}
67+
68+
// Find minimal prefixes for all spec strings so no spec runs are duplicated across chunks.
69+
prefixes := findMinimalWordPrefixes(describes)
70+
sort.Strings(prefixes)
71+
72+
var out string
73+
if opts.printDebug {
74+
out = strings.Join(prefixes, "\n")
75+
} else {
76+
out, err = createChunkRegexp(opts.numChunks, opts.printChunk, prefixes)
77+
if err != nil {
78+
return err
79+
}
80+
}
81+
82+
fmt.Fprint(opts.writer, out)
83+
return nil
84+
}
85+
86+
// TODO: this is hacky because top-level tests may be defined elsewise.
87+
// A better strategy would be to use the output of `ginkgo -noColor -dryRun`
88+
// like https://github.com/operator-framework/operator-lifecycle-manager/pull/1476 does.
89+
var topDescribeRE = regexp.MustCompile(`var _ = Describe\("(.+)", func\(.*`)
90+
91+
func findDescribes(dir string) ([]string, error) {
92+
// Find all Ginkgo specs in dir's test files.
93+
// These can be grouped independently.
94+
describeTable := make(map[string]struct{})
95+
matches, err := filepath.Glob(filepath.Join(dir, "*_test.go"))
96+
if err != nil {
97+
return nil, err
98+
}
99+
for _, match := range matches {
100+
b, err := ioutil.ReadFile(match)
101+
if err != nil {
102+
return nil, err
103+
}
104+
specNames := topDescribeRE.FindAllSubmatch(b, -1)
105+
if len(specNames) == 0 {
106+
log.Printf("%s: found no top level describes, skipping", match)
107+
continue
108+
}
109+
for _, possibleNames := range specNames {
110+
if len(possibleNames) != 2 {
111+
log.Printf("%s: expected to find 2 submatch, found %d:", match, len(possibleNames))
112+
for _, name := range possibleNames {
113+
log.Printf("\t%s\n", string(name))
114+
}
115+
continue
116+
}
117+
describe := strings.TrimSpace(string(possibleNames[1]))
118+
describeTable[describe] = struct{}{}
119+
}
120+
}
121+
122+
describes := make([]string, len(describeTable))
123+
i := 0
124+
for describeKey := range describeTable {
125+
describes[i] = describeKey
126+
i++
127+
}
128+
return describes, nil
129+
}
130+
131+
func createChunkRegexp(numChunks, printChunk int, specs []string) (string, error) {
132+
133+
numSpecs := len(specs)
134+
if numSpecs < numChunks {
135+
return "", fmt.Errorf("have more desired chunks (%d) than specs (%d)", numChunks, numSpecs)
136+
}
137+
138+
// Create chunks of size ceil(number of specs/number of chunks) in alphanumeric order.
139+
// This is deterministic on inputs.
140+
chunks := make([][]string, numChunks)
141+
interval := int(math.Ceil(float64(numSpecs) / float64(numChunks)))
142+
currIdx := 0
143+
for chunkIdx := 0; chunkIdx < numChunks; chunkIdx++ {
144+
nextIdx := int(math.Min(float64(currIdx+interval), float64(numSpecs)))
145+
chunks[chunkIdx] = specs[currIdx:nextIdx]
146+
currIdx = nextIdx
147+
}
148+
149+
chunk := chunks[printChunk]
150+
if len(chunk) == 0 {
151+
// This is a panic because the caller may skip this error, resulting in missed test specs.
152+
panic(fmt.Sprintf("bug: chunk %d has no elements", printChunk))
153+
}
154+
155+
// Write out the regexp to focus chunk specs via `ginkgo -focus <re>`.
156+
var reStr string
157+
if len(chunk) == 1 {
158+
reStr = fmt.Sprintf("%s .*", chunk[0])
159+
} else {
160+
sb := strings.Builder{}
161+
sb.WriteString(chunk[0])
162+
for _, test := range chunk[1:] {
163+
sb.WriteString("|")
164+
sb.WriteString(test)
165+
}
166+
reStr = fmt.Sprintf("(%s) .*", sb.String())
167+
}
168+
169+
return reStr, nil
170+
}
171+
172+
func findMinimalWordPrefixes(specs []string) (prefixes []string) {
173+
174+
// Create a word trie of all spec strings.
175+
t := make(wordTrie)
176+
for _, spec := range specs {
177+
t.push(spec)
178+
}
179+
180+
// Now find the first branch point for each path in the trie by DFS.
181+
for word, node := range t {
182+
var prefixElements []string
183+
next:
184+
if word != "" {
185+
prefixElements = append(prefixElements, word)
186+
}
187+
if len(node.children) == 1 {
188+
for nextWord, nextNode := range node.children {
189+
word, node = nextWord, nextNode
190+
}
191+
goto next
192+
}
193+
// TODO: this might need to be joined by "\s+"
194+
// in case multiple spaces were used in the spec name.
195+
prefixes = append(prefixes, strings.Join(prefixElements, " "))
196+
}
197+
198+
return prefixes
199+
}
200+
201+
// wordTrie is a trie of word nodes, instead of individual characters.
202+
type wordTrie map[string]*wordTrieNode
203+
204+
type wordTrieNode struct {
205+
word string
206+
children map[string]*wordTrieNode
207+
}
208+
209+
// push creates s branch of the trie from each word in s.
210+
func (t wordTrie) push(s string) {
211+
split := strings.Split(s, " ")
212+
213+
curr := &wordTrieNode{word: "", children: t}
214+
for _, sp := range split {
215+
if sp = strings.TrimSpace(sp); sp == "" {
216+
continue
217+
}
218+
next, hasNext := curr.children[sp]
219+
if !hasNext {
220+
next = &wordTrieNode{word: sp, children: make(map[string]*wordTrieNode)}
221+
curr.children[sp] = next
222+
}
223+
curr = next
224+
}
225+
// Add termination node so "foo" and "foo bar" have a branching point of "foo".
226+
curr.children[""] = &wordTrieNode{}
227+
}

0 commit comments

Comments
 (0)