Skip to content

Commit 1b00c77

Browse files
serhalpserhalp
authored
test: fix flakey snapshot normalization logic (#7203)
* test: fix flakey snapshot normalization logic This logic was normalizing integration test snapshots by replacing any sequence of five digits with `88888`. The intent is to handle some randomness in test output. It wasn't working some percentage of the time because randomly selected ports are sometimes _four_ digits. You can see an example in this failed run: https://github.com/netlify/cli/actions/runs/14434953424/job/40474516002?pr=7202#step:10:1143. While I was at it, I made it less much less generic and moved it into the only test suite that relies on it. * test: make normalized snapshot port more explicit
1 parent b60c7d3 commit 1b00c77

File tree

3 files changed

+34
-30
lines changed

3 files changed

+34
-30
lines changed

‎tests/integration/__snapshots__/framework-detection.test.js.snap

+10-10
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,11 @@ exports[`frameworks/framework-detection > should default to process.cwd() and st
99
◈ Running static server from \\"should-default-to-process-cwd-and-static-server\\"
1010
◈ Setting up local development server
1111
12-
◈ Static server listening to 88888
12+
◈ Static server listening to <SNAPSHOT_PORT_NORMALIZED>
1313
1414
┌──────────────────────────────────────────────────┐
1515
│ │
16-
│ ◈ Server now ready on http://localhost:88888 │
16+
│ ◈ Server now ready on http://localhost:<SNAPSHOT_PORT_NORMALIZED> │
1717
│ │
1818
└──────────────────────────────────────────────────┘"
1919
`;
@@ -38,11 +38,11 @@ exports[`frameworks/framework-detection > should filter frameworks with no dev c
3838
◈ Running static server from \\"should-filter-frameworks-with-no-dev-command\\"
3939
◈ Setting up local development server
4040
41-
◈ Static server listening to 88888
41+
◈ Static server listening to <SNAPSHOT_PORT_NORMALIZED>
4242
4343
┌──────────────────────────────────────────────────┐
4444
│ │
45-
│ ◈ Server now ready on http://localhost:88888 │
45+
│ ◈ Server now ready on http://localhost:<SNAPSHOT_PORT_NORMALIZED> │
4646
│ │
4747
└──────────────────────────────────────────────────┘"
4848
`;
@@ -150,11 +150,11 @@ exports[`frameworks/framework-detection > should use static server when --dir fl
150150
◈ Running static server from \\"should-use-static-server-when-dir-flag-is-passed/public\\"
151151
◈ Setting up local development server
152152
153-
◈ Static server listening to 88888
153+
◈ Static server listening to <SNAPSHOT_PORT_NORMALIZED>
154154
155155
┌──────────────────────────────────────────────────┐
156156
│ │
157-
│ ◈ Server now ready on http://localhost:88888 │
157+
│ ◈ Server now ready on http://localhost:<SNAPSHOT_PORT_NORMALIZED> │
158158
│ │
159159
└──────────────────────────────────────────────────┘"
160160
`;
@@ -168,11 +168,11 @@ exports[`frameworks/framework-detection > should use static server when framewor
168168
◈ Running static server from \\"should-use-static-server-when-framework-is-set-to-static\\"
169169
◈ Setting up local development server
170170
171-
◈ Static server listening to 88888
171+
◈ Static server listening to <SNAPSHOT_PORT_NORMALIZED>
172172
173173
┌──────────────────────────────────────────────────┐
174174
│ │
175-
│ ◈ Server now ready on http://localhost:88888 │
175+
│ ◈ Server now ready on http://localhost:<SNAPSHOT_PORT_NORMALIZED> │
176176
│ │
177177
└──────────────────────────────────────────────────┘"
178178
`;
@@ -185,11 +185,11 @@ exports[`frameworks/framework-detection > should warn if using static server and
185185
◈ Running static server from \\"should-warn-if-using-static-server-and-target-port-is-configured/public\\"
186186
◈ Setting up local development server
187187
188-
◈ Static server listening to 88888
188+
◈ Static server listening to <SNAPSHOT_PORT_NORMALIZED>
189189
190190
┌──────────────────────────────────────────────────┐
191191
│ │
192-
│ ◈ Server now ready on http://localhost:88888 │
192+
│ ◈ Server now ready on http://localhost:<SNAPSHOT_PORT_NORMALIZED> │
193193
│ │
194194
└──────────────────────────────────────────────────┘"
195195
`;

‎tests/integration/framework-detection.test.js

+24-18
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,12 @@ import { normalize } from './utils/snapshots.js'
1010

1111
const content = 'Hello World!'
1212

13+
// Normalize random ports
14+
const normalizeSnapshot = (output, opts) =>
15+
normalize(output, opts)
16+
.replaceAll(/localhost:\d+/g, 'localhost:<SNAPSHOT_PORT_NORMALIZED>')
17+
.replaceAll(/listening to \d+/g, 'listening to <SNAPSHOT_PORT_NORMALIZED>')
18+
1319
describe.concurrent('frameworks/framework-detection', () => {
1420
test('should default to process.cwd() and static server', async (t) => {
1521
await withSiteBuilder(t, async (builder) => {
@@ -25,7 +31,7 @@ describe.concurrent('frameworks/framework-detection', () => {
2531
const responseContent = await response.text()
2632

2733
t.expect(responseContent).toEqual(content)
28-
t.expect(normalize(output, { duration: true, filePath: true })).toMatchSnapshot()
34+
t.expect(normalizeSnapshot(output, { duration: true, filePath: true })).toMatchSnapshot()
2935
})
3036
})
3137
})
@@ -44,7 +50,7 @@ describe.concurrent('frameworks/framework-detection', () => {
4450
const responseContent = await response.text()
4551

4652
t.expect(responseContent).toEqual(content)
47-
t.expect(normalize(output, { duration: true, filePath: true })).toMatchSnapshot()
53+
t.expect(normalizeSnapshot(output, { duration: true, filePath: true })).toMatchSnapshot()
4854
})
4955
})
5056
})
@@ -64,7 +70,7 @@ describe.concurrent('frameworks/framework-detection', () => {
6470
const responseContent = await response.text()
6571

6672
t.expect(responseContent).toEqual(content)
67-
t.expect(normalize(output, { duration: true, filePath: true })).toMatchSnapshot()
73+
t.expect(normalizeSnapshot(output, { duration: true, filePath: true })).toMatchSnapshot()
6874
})
6975
})
7076
})
@@ -85,7 +91,7 @@ describe.concurrent('frameworks/framework-detection', () => {
8591
const responseContent = await response.text()
8692

8793
t.expect(responseContent).toEqual(content)
88-
t.expect(normalize(output, { duration: true, filePath: true })).toMatchSnapshot()
94+
t.expect(normalizeSnapshot(output, { duration: true, filePath: true })).toMatchSnapshot()
8995
},
9096
)
9197
})
@@ -102,7 +108,7 @@ describe.concurrent('frameworks/framework-detection', () => {
102108
true,
103109
).catch((error_) => error_)
104110

105-
t.expect(normalize(error.stdout, { duration: true, filePath: true })).toMatchSnapshot()
111+
t.expect(normalizeSnapshot(error.stdout, { duration: true, filePath: true })).toMatchSnapshot()
106112
})
107113
})
108114

@@ -112,7 +118,7 @@ describe.concurrent('frameworks/framework-detection', () => {
112118

113119
// a failure is expected since this is not a true create-react-app project
114120
const error = await withDevServer({ cwd: builder.directory }, () => {}, true).catch((error_) => error_)
115-
t.expect(normalize(error.stdout, { duration: true, filePath: true })).toMatchSnapshot()
121+
t.expect(normalizeSnapshot(error.stdout, { duration: true, filePath: true })).toMatchSnapshot()
116122
})
117123
})
118124

@@ -121,7 +127,7 @@ describe.concurrent('frameworks/framework-detection', () => {
121127
await builder.withNetlifyToml({ config: { dev: { framework: 'to-infinity-and-beyond-js' } } }).build()
122128

123129
const error = await withDevServer({ cwd: builder.directory }, () => {}, true).catch((error_) => error_)
124-
t.expect(normalize(error.stdout, { duration: true, filePath: true })).toMatchSnapshot()
130+
t.expect(normalizeSnapshot(error.stdout, { duration: true, filePath: true })).toMatchSnapshot()
125131
})
126132
})
127133

@@ -135,7 +141,7 @@ describe.concurrent('frameworks/framework-detection', () => {
135141

136142
// a failure is expected since this is not a true create-react-app project
137143
const error = await withDevServer({ cwd: builder.directory }, () => {}, true).catch((error_) => error_)
138-
t.expect(normalize(error.stdout, { duration: true, filePath: true })).toMatchSnapshot()
144+
t.expect(normalizeSnapshot(error.stdout, { duration: true, filePath: true })).toMatchSnapshot()
139145
})
140146
})
141147

@@ -148,7 +154,7 @@ describe.concurrent('frameworks/framework-detection', () => {
148154
() => {},
149155
true,
150156
).catch((error_) => error_)
151-
t.expect(normalize(error.stdout, { duration: true, filePath: true })).toMatchSnapshot()
157+
t.expect(normalizeSnapshot(error.stdout, { duration: true, filePath: true })).toMatchSnapshot()
152158
})
153159
})
154160

@@ -161,7 +167,7 @@ describe.concurrent('frameworks/framework-detection', () => {
161167
() => {},
162168
true,
163169
).catch((error_) => error_)
164-
t.expect(normalize(error.stdout, { duration: true, filePath: true })).toMatchSnapshot()
170+
t.expect(normalizeSnapshot(error.stdout, { duration: true, filePath: true })).toMatchSnapshot()
165171
})
166172
})
167173

@@ -174,7 +180,7 @@ describe.concurrent('frameworks/framework-detection', () => {
174180
() => {},
175181
true,
176182
).catch((error_) => error_)
177-
t.expect(normalize(error.stdout, { duration: true, filePath: true })).toMatchSnapshot()
183+
t.expect(normalizeSnapshot(error.stdout, { duration: true, filePath: true })).toMatchSnapshot()
178184
})
179185
})
180186

@@ -198,7 +204,7 @@ describe.concurrent('frameworks/framework-detection', () => {
198204
true,
199205
).catch((error_) => error_)
200206

201-
t.expect(normalize(error.stdout, { duration: true, filePath: true })).toMatchSnapshot()
207+
t.expect(normalizeSnapshot(error.stdout, { duration: true, filePath: true })).toMatchSnapshot()
202208
})
203209
})
204210

@@ -232,7 +238,7 @@ describe.concurrent('frameworks/framework-detection', () => {
232238
await childProcess
233239
}
234240
const error = await asyncErrorBlock().catch((error_) => error_)
235-
t.expect(normalize(error.stdout, { duration: true, filePath: true })).toMatchSnapshot()
241+
t.expect(normalizeSnapshot(error.stdout, { duration: true, filePath: true })).toMatchSnapshot()
236242
})
237243
})
238244

@@ -259,7 +265,7 @@ describe.concurrent('frameworks/framework-detection', () => {
259265
}
260266
const error = await asyncErrorBlock().catch((error_) => error_)
261267
t.expect(
262-
normalize(error.stdout, { duration: true, filePath: true }).includes(
268+
normalizeSnapshot(error.stdout, { duration: true, filePath: true }).includes(
263269
'Detected commands for: Gatsby, Create React App. Update your settings to specify which to use. Refer to https://ntl.fyi/dev-monorepo for more information.',
264270
),
265271
)
@@ -278,7 +284,7 @@ describe.concurrent('frameworks/framework-detection', () => {
278284
true,
279285
).catch((error_) => error_)
280286

281-
t.expect(normalize(error.stdout, { duration: true, filePath: true })).toMatchSnapshot()
287+
t.expect(normalizeSnapshot(error.stdout, { duration: true, filePath: true })).toMatchSnapshot()
282288
})
283289
})
284290

@@ -299,7 +305,7 @@ describe.concurrent('frameworks/framework-detection', () => {
299305
const responseContent = await response.text()
300306

301307
t.expect(responseContent).toEqual(content)
302-
t.expect(normalize(output, { duration: true, filePath: true })).toMatchSnapshot()
308+
t.expect(normalizeSnapshot(output, { duration: true, filePath: true })).toMatchSnapshot()
303309
})
304310
})
305311
})
@@ -317,7 +323,7 @@ describe.concurrent('frameworks/framework-detection', () => {
317323

318324
// a failure is expected since this is not a true Gatsby project
319325
const error = await withDevServer({ cwd: builder.directory }, () => {}, true).catch((error_) => error_)
320-
t.expect(normalize(error.stdout, { duration: true, filePath: true })).toMatchSnapshot()
326+
t.expect(normalizeSnapshot(error.stdout, { duration: true, filePath: true })).toMatchSnapshot()
321327
})
322328
})
323329

@@ -395,7 +401,7 @@ describe.concurrent('frameworks/framework-detection', () => {
395401
const responseJson = await response.json()
396402
t.expect(responseJson).toStrictEqual({ CONTEXT_CHECK: 'PRODUCTION' })
397403

398-
const normalizedText = normalize(output, { duration: true, filePath: true })
404+
const normalizedText = normalizeSnapshot(output, { duration: true, filePath: true })
399405
t.expect(
400406
normalizedText.includes(
401407
`Changes will not be hot-reloaded, so if you need to rebuild your site you must exit and run 'netlify serve' again`,

‎tests/integration/utils/snapshots.js

-2
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,6 @@ const baseNormalizers = [
22
// Information about the package and the OS
33
{ pattern: /netlify-cli\/.+node-.+/g, value: 'netlify-cli/test-version test-os test-node-version' },
44
{ pattern: /@netlify\/build (\d+\.\d+\.\d+)/g, value: '@netlify/build 0.0.0' },
5-
// normalize random ports
6-
{ pattern: /\d{5}/g, value: '88888' },
75
// windows specific
86
{ pattern: /\\/gu, value: '/' },
97
{ pattern: /\r\n/gu, value: '\n' },

0 commit comments

Comments
 (0)