Skip to content

Commit a2c5f6e

Browse files
TimBeyerTimBeyer
authored
perf: various performance improvements (#4959)
* perf: speed up `splitLast` function * perf: skip function wrapping if profiler is disabled * perf: index files into a tree structure to get files for a subdirectory more quickly * perf: increase max-semi-space-size for less GC interruptions at the cost of some more memory See https://github.com/nodejs/node/blob/main/doc/api/cli.md#useful-v8-options Also see https://www.alibabacloud.com/blog/better-node-application-performance-through-gc-optimization_595119 and nodejs/node#42511 for some details on impact * refactor: replace Bluebird with native Promises (part 1) * refactor: replace bluebird with native promises (part 2) * refactor: replace bluebird with native promises (part 3) * chore: make the linter happy after all the bluebird replacements * fix: splitLast behavior with no index found * chore: linter again * chore: cleanup file-tree and add header * refactor: replace pLimit with pMap
1 parent 7f67071 commit a2c5f6e

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

80 files changed

+1887
-1623
lines changed

‎bin/garden

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,4 @@
22

33
repo_root=$(cd `dirname $0` && cd .. && pwd)
44

5-
node --max-old-space-size=4096 ${repo_root}/cli/bin/garden "$@"
5+
node --max-old-space-size=4096 --max-semi-space-size=64 ${repo_root}/cli/bin/garden "$@"

‎cli/bin/garden-debug

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
1-
#!/usr/bin/env node --inspect --stack-trace-limit=1000
1+
#!/usr/bin/env node --inspect --stack-trace-limit=1000 --max-semi-space-size=64
22

33
require("./garden")

‎cli/src/add-version-files.ts

+17-16
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@ import { GitHandler } from "@garden-io/core/build/src/vcs/git"
1010
import { Garden } from "@garden-io/core/build/src/garden"
1111
import { LogLevel, RootLogger } from "@garden-io/core/build/src/logger/logger"
1212
import { resolve, relative } from "path"
13-
import Bluebird from "bluebird"
1413
import { STATIC_DIR, GARDEN_VERSIONFILE_NAME } from "@garden-io/core/build/src/constants"
1514
import { writeTreeVersionFile } from "@garden-io/core/build/src/vcs/vcs"
1615
import { TreeCache } from "@garden-io/core/build/src/cache"
@@ -28,24 +27,26 @@ async function addVersionFiles() {
2827

2928
const moduleConfigs = await garden.getRawModuleConfigs()
3029

31-
return Bluebird.map(moduleConfigs, async (config) => {
32-
const path = config.path
33-
const versionFilePath = resolve(path, GARDEN_VERSIONFILE_NAME)
30+
return Promise.all(
31+
moduleConfigs.map(async (config) => {
32+
const path = config.path
33+
const versionFilePath = resolve(path, GARDEN_VERSIONFILE_NAME)
3434

35-
const vcsHandler = new GitHandler({
36-
garden,
37-
projectRoot: STATIC_DIR,
38-
gardenDirPath: garden.gardenDirPath,
39-
ignoreFile: garden.dotIgnoreFile,
40-
cache: new TreeCache(),
41-
})
42-
const treeVersion = await vcsHandler.getTreeVersion({ log: garden.log, projectName: garden.projectName, config })
35+
const vcsHandler = new GitHandler({
36+
garden,
37+
projectRoot: STATIC_DIR,
38+
gardenDirPath: garden.gardenDirPath,
39+
ignoreFile: garden.dotIgnoreFile,
40+
cache: new TreeCache(),
41+
})
42+
const treeVersion = await vcsHandler.getTreeVersion({ log: garden.log, projectName: garden.projectName, config })
4343

44-
// eslint-disable-next-line no-console
45-
console.log(`${config.name} -> ${relative(STATIC_DIR, versionFilePath)}`)
44+
// eslint-disable-next-line no-console
45+
console.log(`${config.name} -> ${relative(STATIC_DIR, versionFilePath)}`)
4646

47-
return writeTreeVersionFile(path, treeVersion)
48-
})
47+
return writeTreeVersionFile(path, treeVersion)
48+
})
49+
)
4950
}
5051

5152
if (require.main === module) {

‎cli/src/build-pkg.ts

+51-46
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@
99
import chalk from "chalk"
1010
import { getAbi } from "node-abi"
1111
import { resolve, relative, join } from "path"
12-
import Bluebird from "bluebird"
1312
import { STATIC_DIR, GARDEN_CLI_ROOT, GARDEN_CORE_ROOT } from "@garden-io/core/build/src/constants"
1413
import { remove, mkdirp, copy, writeFile } from "fs-extra"
1514
import { exec, getPackageVersion, sleep } from "@garden-io/core/build/src/util/util"
@@ -29,7 +28,7 @@ const pkgPath = resolve(repoRoot, "cli", "node_modules", ".bin", "pkg")
2928
const distPath = resolve(repoRoot, "dist")
3029

3130
// Allow larger heap size than default
32-
const nodeOptions = ["max-old-space-size=4096"]
31+
const nodeOptions = ["max-old-space-size=4096", "max-semi-space-size=64"]
3332

3433
/* eslint-disable no-console */
3534

@@ -95,48 +94,52 @@ async function buildBinaries(args: string[]) {
9594
const workspaces = JSON.parse(JSON.parse(res).data)
9695

9796
console.log(chalk.cyan("Copying packages"))
98-
await Bluebird.map(Object.entries(workspaces), async ([name, info]: [string, any]) => {
99-
const sourcePath = resolve(repoRoot, info.location)
100-
const targetPath = resolve(tmpDirPath, info.location)
101-
await remove(targetPath)
102-
await mkdirp(targetPath)
103-
await exec("rsync", [
104-
"-r",
105-
"-L",
106-
"--exclude=node_modules",
107-
"--exclude=tmp",
108-
"--exclude=test",
109-
sourcePath,
110-
resolve(targetPath, ".."),
111-
])
112-
113-
console.log(chalk.green(" ✓ " + name))
114-
})
97+
await Promise.all(
98+
Object.entries(workspaces).map(async ([name, info]: [string, any]) => {
99+
const sourcePath = resolve(repoRoot, info.location)
100+
const targetPath = resolve(tmpDirPath, info.location)
101+
await remove(targetPath)
102+
await mkdirp(targetPath)
103+
await exec("rsync", [
104+
"-r",
105+
"-L",
106+
"--exclude=node_modules",
107+
"--exclude=tmp",
108+
"--exclude=test",
109+
sourcePath,
110+
resolve(targetPath, ".."),
111+
])
112+
113+
console.log(chalk.green(" ✓ " + name))
114+
})
115+
)
115116

116117
// Edit all the packages to have them directly link any internal dependencies
117118
console.log(chalk.cyan("Modifying package.json files for direct installation"))
118-
await Bluebird.map(Object.entries(workspaces), async ([name, info]: [string, any]) => {
119-
const packageRoot = resolve(tmpDirPath, info.location)
120-
const packageJsonPath = resolve(packageRoot, "package.json")
121-
const packageJson = require(packageJsonPath)
122-
123-
for (const depName of info.workspaceDependencies) {
124-
const depInfo = workspaces[depName]
125-
const targetRoot = resolve(tmpDirPath, depInfo.location)
126-
const relPath = relative(packageRoot, targetRoot)
127-
packageJson.dependencies[depName] = "file:" + relPath
128-
}
129-
130-
if (version === "edge" || version.startsWith("edge-")) {
131-
const gitHash = await exec("git", ["rev-parse", "--short", "HEAD"])
132-
packageJson.version = `${packageJson.version}-${version}-${gitHash.stdout}`
133-
console.log("Set package version to " + packageJson.version)
134-
}
135-
136-
await writeFile(packageJsonPath, JSON.stringify(packageJson, null, 2))
137-
138-
console.log(chalk.green(" ✓ " + name))
139-
})
119+
await Promise.all(
120+
Object.entries(workspaces).map(async ([name, info]: [string, any]) => {
121+
const packageRoot = resolve(tmpDirPath, info.location)
122+
const packageJsonPath = resolve(packageRoot, "package.json")
123+
const packageJson = require(packageJsonPath)
124+
125+
for (const depName of info.workspaceDependencies) {
126+
const depInfo = workspaces[depName]
127+
const targetRoot = resolve(tmpDirPath, depInfo.location)
128+
const relPath = relative(packageRoot, targetRoot)
129+
packageJson.dependencies[depName] = "file:" + relPath
130+
}
131+
132+
if (version === "edge" || version.startsWith("edge-")) {
133+
const gitHash = await exec("git", ["rev-parse", "--short", "HEAD"])
134+
packageJson.version = `${packageJson.version}-${version}-${gitHash.stdout}`
135+
console.log("Set package version to " + packageJson.version)
136+
}
137+
138+
await writeFile(packageJsonPath, JSON.stringify(packageJson, null, 2))
139+
140+
console.log(chalk.green(" ✓ " + name))
141+
})
142+
)
140143

141144
// Run yarn install in the cli package
142145
console.log(chalk.cyan("Installing packages in @garden-io/cli package"))
@@ -146,11 +149,13 @@ async function buildBinaries(args: string[]) {
146149
// Run pkg and pack up each platform binary
147150
console.log(chalk.cyan("Packaging garden binaries"))
148151

149-
await Bluebird.map(Object.entries(selected), async ([targetName, spec]) => {
150-
await spec.handler({ targetName, sourcePath: cliPath, pkgType: spec.pkgType, version })
151-
await sleep(5000) // Work around concurrency bug in pkg...
152-
console.log(chalk.green(" ✓ " + targetName))
153-
})
152+
await Promise.all(
153+
Object.entries(selected).map(async ([targetName, spec]) => {
154+
await spec.handler({ targetName, sourcePath: cliPath, pkgType: spec.pkgType, version })
155+
await sleep(5000) // Work around concurrency bug in pkg...
156+
console.log(chalk.green(" ✓ " + targetName))
157+
})
158+
)
154159

155160
console.log(chalk.green.bold("Done!"))
156161
}

‎core/package.json

+5-3
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,6 @@
4242
"async": "^3.2.4",
4343
"@scg82/exit-hook": "^3.4.1",
4444
"async-lock": "^1.4.0",
45-
"bluebird": "^3.7.2",
4645
"certpem": "^1.1.3",
4746
"chalk": "^4.1.2",
4847
"chokidar": "^3.5.3",
@@ -113,8 +112,12 @@
113112
"normalize-path": "^3.0.0",
114113
"normalize-url": "^5.3.1",
115114
"open": "^8.4.2",
115+
"p-filter": "^2.1.0",
116116
"p-queue": "^6.0.0",
117117
"p-retry": "^4.6.2",
118+
"p-limit": "^3.1.0",
119+
"p-map": "^4.0.0",
120+
"p-props": "^4.0.0",
118121
"parse-duration": "^1.0.2",
119122
"parse-git-config": "^3.0.0",
120123
"path-is-inside": "^1.0.2",
@@ -166,7 +169,6 @@
166169
"@types/analytics-node": "^3.1.10",
167170
"@types/async": "^3.2.18",
168171
"@types/async-lock": "^1.4.0",
169-
"@types/bluebird": "^3.5.38",
170172
"@types/chai": "^4.3.4",
171173
"@types/circular-json": "^0.4.0",
172174
"@types/cross-spawn": "^6.0.2",
@@ -278,4 +280,4 @@
278280
]
279281
},
280282
"gitHead": "b0647221a4d2ff06952bae58000b104215aed922"
281-
}
283+
}

‎core/src/actions/helpers.ts

+64-57
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
77
*/
88

9-
import Bluebird from "bluebird"
109
import chalk from "chalk"
1110
import { fromPairs, memoize } from "lodash"
1211
import { joi } from "../config/common"
@@ -158,22 +157,24 @@ export async function getDeployStatusPayloads({
158157
const actions = graph.getDeploys()
159158

160159
return fromPairs(
161-
await Bluebird.map(actions, async (action) => {
162-
const startedAt = new Date().toISOString()
163-
const actionLog = createActionLog({ log, actionName: action.name, actionKind: action.kind })
164-
const { result } = await router.deploy.getStatus({ action, log: actionLog, graph })
160+
await Promise.all(
161+
actions.map(async (action) => {
162+
const startedAt = new Date().toISOString()
163+
const actionLog = createActionLog({ log, actionName: action.name, actionKind: action.kind })
164+
const { result } = await router.deploy.getStatus({ action, log: actionLog, graph })
165165

166-
const payload = makeActionCompletePayload({
167-
result,
168-
operation: "getStatus",
169-
startedAt,
170-
force: false,
171-
action,
172-
sessionId,
173-
}) as ActionStatusPayload<DeployStatusForEventPayload>
166+
const payload = makeActionCompletePayload({
167+
result,
168+
operation: "getStatus",
169+
startedAt,
170+
force: false,
171+
action,
172+
sessionId,
173+
}) as ActionStatusPayload<DeployStatusForEventPayload>
174174

175-
return [action.name, payload]
176-
})
175+
return [action.name, payload]
176+
})
177+
)
177178
)
178179
}
179180

@@ -191,22 +192,24 @@ export async function getBuildStatusPayloads({
191192
const actions = graph.getBuilds()
192193

193194
return fromPairs(
194-
await Bluebird.map(actions, async (action) => {
195-
const startedAt = new Date().toISOString()
196-
const actionLog = createActionLog({ log, actionName: action.name, actionKind: action.kind })
197-
const { result } = await router.build.getStatus({ action, log: actionLog, graph })
195+
await Promise.all(
196+
actions.map(async (action) => {
197+
const startedAt = new Date().toISOString()
198+
const actionLog = createActionLog({ log, actionName: action.name, actionKind: action.kind })
199+
const { result } = await router.build.getStatus({ action, log: actionLog, graph })
198200

199-
const payload = makeActionCompletePayload({
200-
result,
201-
operation: "getStatus",
202-
startedAt,
203-
force: false,
204-
action,
205-
sessionId,
206-
}) as ActionStatusPayload<BuildStatusForEventPayload>
201+
const payload = makeActionCompletePayload({
202+
result,
203+
operation: "getStatus",
204+
startedAt,
205+
force: false,
206+
action,
207+
sessionId,
208+
}) as ActionStatusPayload<BuildStatusForEventPayload>
207209

208-
return [action.name, payload]
209-
})
210+
return [action.name, payload]
211+
})
212+
)
210213
)
211214
}
212215

@@ -224,20 +227,22 @@ export async function getTestStatusPayloads({
224227
const actions = graph.getTests()
225228

226229
return fromPairs(
227-
await Bluebird.map(actions, async (action) => {
228-
const actionLog = createActionLog({ log, actionName: action.name, actionKind: action.kind })
229-
const startedAt = new Date().toISOString()
230-
const { result } = await router.test.getResult({ action, log: actionLog, graph })
231-
const payload = makeActionCompletePayload({
232-
result,
233-
operation: "getStatus",
234-
startedAt,
235-
force: false,
236-
action,
237-
sessionId,
238-
}) as ActionStatusPayload<RunStatusForEventPayload>
239-
return [action.name, payload]
240-
})
230+
await Promise.all(
231+
actions.map(async (action) => {
232+
const actionLog = createActionLog({ log, actionName: action.name, actionKind: action.kind })
233+
const startedAt = new Date().toISOString()
234+
const { result } = await router.test.getResult({ action, log: actionLog, graph })
235+
const payload = makeActionCompletePayload({
236+
result,
237+
operation: "getStatus",
238+
startedAt,
239+
force: false,
240+
action,
241+
sessionId,
242+
}) as ActionStatusPayload<RunStatusForEventPayload>
243+
return [action.name, payload]
244+
})
245+
)
241246
)
242247
}
243248

@@ -255,21 +260,23 @@ export async function getRunStatusPayloads({
255260
const actions = graph.getRuns()
256261

257262
return fromPairs(
258-
await Bluebird.map(actions, async (action) => {
259-
const actionLog = createActionLog({ log, actionName: action.name, actionKind: action.kind })
260-
const startedAt = new Date().toISOString()
261-
const { result } = await router.run.getResult({ action, log: actionLog, graph })
263+
await Promise.all(
264+
actions.map(async (action) => {
265+
const actionLog = createActionLog({ log, actionName: action.name, actionKind: action.kind })
266+
const startedAt = new Date().toISOString()
267+
const { result } = await router.run.getResult({ action, log: actionLog, graph })
262268

263-
const payload = makeActionCompletePayload({
264-
result,
265-
operation: "getStatus",
266-
startedAt,
267-
force: false,
268-
action,
269-
sessionId,
270-
}) as ActionStatusPayload<RunStatusForEventPayload>
269+
const payload = makeActionCompletePayload({
270+
result,
271+
operation: "getStatus",
272+
startedAt,
273+
force: false,
274+
action,
275+
sessionId,
276+
}) as ActionStatusPayload<RunStatusForEventPayload>
271277

272-
return [action.name, payload]
273-
})
278+
return [action.name, payload]
279+
})
280+
)
274281
)
275282
}

0 commit comments

Comments
 (0)