diff --git a/.changeset/plenty-spoons-build.md b/.changeset/plenty-spoons-build.md new file mode 100644 index 0000000000..0fbfac64c4 --- /dev/null +++ b/.changeset/plenty-spoons-build.md @@ -0,0 +1,6 @@ +--- +"trigger.dev": patch +"@trigger.dev/core": patch +--- + +Prettier and more specific errors with links to docs diff --git a/apps/coordinator/src/index.ts b/apps/coordinator/src/index.ts index d52edfb3aa..238dd45db7 100644 --- a/apps/coordinator/src/index.ts +++ b/apps/coordinator/src/index.ts @@ -401,13 +401,16 @@ class TaskCoordinator { success: true, }; } catch (error) { - logger.error("Error while waiting for checkpointable state", { error }); + logger.error("Error while waiting for checkpointable state", { + error, + runId: socket.data.runId, + }); if (error instanceof CheckpointReadinessTimeoutError) { - await crashRun({ - name: error.name, - message: `Failed to become checkpointable in ${CHECKPOINTABLE_TIMEOUT_SECONDS}s for ${reason}`, - }); + logger.error( + `Failed to become checkpointable in ${CHECKPOINTABLE_TIMEOUT_SECONDS}s for ${reason}`, + { runId: socket.data.runId } + ); return { success: false, @@ -490,7 +493,7 @@ class TaskCoordinator { updateAttemptFriendlyId(executionAck.payload.execution.attempt.id); updateAttemptNumber(executionAck.payload.execution.attempt.number); } catch (error) { - logger.error("Error", { error }); + logger.error("READY_FOR_EXECUTION error", { error, runId: socket.data.runId }); await crashRun({ name: "ReadyForExecutionError", @@ -524,7 +527,10 @@ class TaskCoordinator { } if (!lazyAttempt.success) { - logger.error("failed to get lazy attempt payload", { runId: socket.data.runId }); + logger.error("failed to get lazy attempt payload", { + runId: socket.data.runId, + reason: lazyAttempt.reason, + }); await crashRun({ name: "ReadyForLazyAttemptError", @@ -546,7 +552,7 @@ class TaskCoordinator { return; } - logger.error("Error", { error }); + logger.error("READY_FOR_LAZY_ATTEMPT error", { error, runId: socket.data.runId }); await crashRun({ name: "ReadyForLazyAttemptError", @@ -1004,7 +1010,10 @@ class TaskCoordinator { }); if (!createAttempt?.success) { - logger.debug("no ack while creating attempt", message); + logger.debug("no ack while creating attempt", { + runId: message.runId, + reason: createAttempt?.reason, + }); callback({ success: false, reason: createAttempt?.reason }); return; } diff --git a/apps/kubernetes-provider/src/taskMonitor.ts b/apps/kubernetes-provider/src/taskMonitor.ts index fc142efbd3..9d54b401ee 100644 --- a/apps/kubernetes-provider/src/taskMonitor.ts +++ b/apps/kubernetes-provider/src/taskMonitor.ts @@ -3,13 +3,14 @@ import { SimpleLogger } from "@trigger.dev/core/v3/apps"; import { EXIT_CODE_ALREADY_HANDLED, EXIT_CODE_CHILD_NONZERO } from "@trigger.dev/core/v3/apps"; import { setTimeout } from "timers/promises"; import PQueue from "p-queue"; -import type { Prettify } from "@trigger.dev/core/v3"; +import { TaskRunErrorCodes, type Prettify, type TaskRunInternalError } from "@trigger.dev/core/v3"; type FailureDetails = Prettify<{ exitCode: number; reason: string; logs: string; overrideCompletion: boolean; + errorCode: TaskRunInternalError["code"]; }>; type IndexFailureHandler = (deploymentId: string, details: FailureDetails) => Promise; @@ -160,18 +161,23 @@ export class TaskMonitor { let reason = rawReason || "Unknown error"; let logs = rawLogs || ""; let overrideCompletion = false; + let errorCode: TaskRunInternalError["code"] = TaskRunErrorCodes.POD_UNKNOWN_ERROR; switch (rawReason) { case "Error": reason = "Unknown error."; + errorCode = TaskRunErrorCodes.POD_UNKNOWN_ERROR; break; case "Evicted": if (message.startsWith("Pod ephemeral local storage usage")) { reason = "Storage limit exceeded."; + errorCode = TaskRunErrorCodes.DISK_SPACE_EXCEEDED; } else if (message) { reason = `Evicted: ${message}`; + errorCode = TaskRunErrorCodes.POD_EVICTED; } else { reason = "Evicted for unknown reason."; + errorCode = TaskRunErrorCodes.POD_EVICTED; } if (logs.startsWith("failed to try resolving symlinks")) { @@ -183,6 +189,7 @@ export class TaskMonitor { reason = `${ exitCode === EXIT_CODE_CHILD_NONZERO ? "Child process" : "Parent process" } ran out of memory! Try choosing a machine preset with more memory for this task.`; + errorCode = TaskRunErrorCodes.TASK_PROCESS_OOM_KILLED; break; default: break; @@ -193,6 +200,7 @@ export class TaskMonitor { reason, logs, overrideCompletion, + errorCode, } satisfies FailureDetails; const app = pod.metadata?.labels?.app; diff --git a/apps/webapp/app/components/runs/v3/RunInspector.tsx b/apps/webapp/app/components/runs/v3/RunInspector.tsx index 16e98e5b46..dcbf807835 100644 --- a/apps/webapp/app/components/runs/v3/RunInspector.tsx +++ b/apps/webapp/app/components/runs/v3/RunInspector.tsx @@ -1,6 +1,11 @@ import { CheckIcon, ClockIcon, CloudArrowDownIcon, QueueListIcon } from "@heroicons/react/20/solid"; import { Link } from "@remix-run/react"; -import { formatDuration, formatDurationMilliseconds, TaskRunError } from "@trigger.dev/core/v3"; +import { + formatDuration, + formatDurationMilliseconds, + TaskRunError, + taskRunErrorEnhancer, +} from "@trigger.dev/core/v3"; import { useEffect } from "react"; import { useTypedFetcher } from "remix-typedjson"; import { ExitIcon } from "~/assets/icons/ExitIcon"; @@ -553,7 +558,9 @@ function RunTimeline({ run }: { run: RawRun }) { } function RunError({ error }: { error: TaskRunError }) { - switch (error.type) { + const enhancedError = taskRunErrorEnhancer(error); + + switch (enhancedError.type) { case "STRING_ERROR": case "CUSTOM_ERROR": { return ( @@ -561,7 +568,7 @@ function RunError({ error }: { error: TaskRunError }) { @@ -569,16 +576,21 @@ function RunError({ error }: { error: TaskRunError }) { } case "BUILT_IN_ERROR": case "INTERNAL_ERROR": { - const name = "name" in error ? error.name : error.code; + const name = "name" in enhancedError ? enhancedError.name : enhancedError.code; return (
{name} - {error.message && {error.message}} - {error.stackTrace && ( + {enhancedError.message && {enhancedError.message}} + {enhancedError.link && ( + + {enhancedError.link.name} + + )} + {enhancedError.stackTrace && ( )} diff --git a/apps/webapp/app/components/runs/v3/SpanEvents.tsx b/apps/webapp/app/components/runs/v3/SpanEvents.tsx index ccc93155f0..c6135f3f9e 100644 --- a/apps/webapp/app/components/runs/v3/SpanEvents.tsx +++ b/apps/webapp/app/components/runs/v3/SpanEvents.tsx @@ -1,4 +1,5 @@ import { + exceptionEventEnhancer, isExceptionSpanEvent, type ExceptionEventProperties, type SpanEvent as OtelSpanEvent, @@ -64,19 +65,26 @@ export function SpanEventError({ spanEvent: OtelSpanEvent; exception: ExceptionEventProperties; }) { + const enhancedException = exceptionEventEnhancer(exception); + return (
- {exception.message && {exception.message}} - {exception.stacktrace && ( + {enhancedException.message && {enhancedException.message}} + {enhancedException.link && ( + + {enhancedException.link.name} + + )} + {enhancedException.stacktrace && ( )} diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam.spans.$spanParam/route.tsx b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam.spans.$spanParam/route.tsx index b1d3a7f11e..afb94a40bc 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam.spans.$spanParam/route.tsx +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam.spans.$spanParam/route.tsx @@ -6,6 +6,7 @@ import { formatDurationMilliseconds, nanosecondsToMilliseconds, TaskRunError, + taskRunErrorEnhancer, } from "@trigger.dev/core/v3"; import { ReactNode, useEffect } from "react"; import { typedjson, useTypedFetcher } from "remix-typedjson"; @@ -933,12 +934,14 @@ function RunTimelineLine({ title, state }: RunTimelineLineProps) { } function RunError({ error }: { error: TaskRunError }) { - switch (error.type) { + const enhancedError = taskRunErrorEnhancer(error); + + switch (enhancedError.type) { case "STRING_ERROR": return (
Error - {error.raw} + {enhancedError.raw}
); case "CUSTOM_ERROR": { @@ -947,7 +950,7 @@ function RunError({ error }: { error: TaskRunError }) {
@@ -955,16 +958,21 @@ function RunError({ error }: { error: TaskRunError }) { } case "BUILT_IN_ERROR": case "INTERNAL_ERROR": { - const name = "name" in error ? error.name : error.code; + const name = "name" in enhancedError ? enhancedError.name : enhancedError.code; return (
{name} - {error.message && {error.message}} - {error.stackTrace && ( + {enhancedError.message && {enhancedError.message}} + {enhancedError.link && ( + + {enhancedError.link.name} + + )} + {enhancedError.stackTrace && ( )} diff --git a/apps/webapp/app/v3/handleSocketIo.server.ts b/apps/webapp/app/v3/handleSocketIo.server.ts index 5b2c600166..065d2bc168 100644 --- a/apps/webapp/app/v3/handleSocketIo.server.ts +++ b/apps/webapp/app/v3/handleSocketIo.server.ts @@ -188,7 +188,7 @@ function createCoordinatorNamespace(io: Server) { const environment = await findEnvironmentById(message.envId); if (!environment) { - logger.error("Environment not found", { id: message.envId }); + logger.error("CREATE_TASK_RUN_ATTEMPT: Environment not found", message); return { success: false, reason: "Environment not found" }; } @@ -198,16 +198,14 @@ function createCoordinatorNamespace(io: Server) { const payload = await sharedQueueTasks.getExecutionPayloadFromAttempt(attempt.id, true); if (!payload) { - logger.error("Failed to retrieve payload after attempt creation", { - id: message.envId, - }); + logger.error("Failed to retrieve payload after attempt creation", message); return { success: false, reason: "Failed to retrieve payload" }; } return { success: true, executionPayload: payload }; } catch (error) { logger.error("Error while creating attempt", { - runId: message.runId, + ...message, error, }); return { success: false }; diff --git a/apps/webapp/app/v3/marqs/sharedQueueConsumer.server.ts b/apps/webapp/app/v3/marqs/sharedQueueConsumer.server.ts index 92762629e2..a663011e4a 100644 --- a/apps/webapp/app/v3/marqs/sharedQueueConsumer.server.ts +++ b/apps/webapp/app/v3/marqs/sharedQueueConsumer.server.ts @@ -1160,7 +1160,7 @@ class SharedQueueTasks { const environment = await findEnvironmentById(envId); if (!environment) { - logger.error("Environment not found", { id: envId }); + logger.error("getLazyAttemptPayload: Environment not found", { runId, envId }); return; } @@ -1182,7 +1182,7 @@ class SharedQueueTasks { }); if (!run) { - logger.error("Run not found", { id: runId, envId }); + logger.error("getLazyAttemptPayload: Run not found", { runId, envId }); return; } diff --git a/apps/webapp/app/v3/services/crashTaskRun.server.ts b/apps/webapp/app/v3/services/crashTaskRun.server.ts index 9556699a5b..e71c3717b1 100644 --- a/apps/webapp/app/v3/services/crashTaskRun.server.ts +++ b/apps/webapp/app/v3/services/crashTaskRun.server.ts @@ -5,7 +5,7 @@ import { BaseService } from "./baseService.server"; import { logger } from "~/services/logger.server"; import { AuthenticatedEnvironment } from "~/services/apiAuth.server"; import { CRASHABLE_ATTEMPT_STATUSES, isCrashableRunStatus } from "../taskStatus"; -import { sanitizeError } from "@trigger.dev/core/v3"; +import { sanitizeError, TaskRunInternalError } from "@trigger.dev/core/v3"; import { FinalizeTaskRunService } from "./finalizeTaskRun.server"; export type CrashTaskRunServiceOptions = { @@ -15,6 +15,7 @@ export type CrashTaskRunServiceOptions = { crashAttempts?: boolean; crashedAt?: Date; overrideCompletion?: boolean; + errorCode?: TaskRunInternalError["code"]; }; export class CrashTaskRunService extends BaseService { @@ -26,6 +27,8 @@ export class CrashTaskRunService extends BaseService { ...options, }; + logger.debug("CrashTaskRunService.call", { runId, opts }); + const taskRun = await this._prisma.taskRun.findFirst({ where: { id: runId, @@ -71,7 +74,7 @@ export class CrashTaskRunService extends BaseService { attemptStatus: "FAILED", error: { type: "INTERNAL_ERROR", - code: "TASK_RUN_CRASHED", + code: opts.errorCode ?? "TASK_RUN_CRASHED", message: opts.reason, stackTrace: opts.logs, }, @@ -129,6 +132,7 @@ export class CrashTaskRunService extends BaseService { error: { reason: string; logs?: string; + code?: TaskRunInternalError["code"]; } ) { return await this.traceWithEnv("failAttempt()", environment, async (span) => { @@ -146,7 +150,7 @@ export class CrashTaskRunService extends BaseService { completedAt: failedAt, error: sanitizeError({ type: "INTERNAL_ERROR", - code: "TASK_RUN_CRASHED", + code: error.code ?? "TASK_RUN_CRASHED", message: error.reason, stackTrace: error.logs, }), diff --git a/packages/cli-v3/src/deploy/logs.ts b/packages/cli-v3/src/deploy/logs.ts index d0ad58ae9b..cebcea0128 100644 --- a/packages/cli-v3/src/deploy/logs.ts +++ b/packages/cli-v3/src/deploy/logs.ts @@ -3,7 +3,7 @@ import { writeFile } from "node:fs/promises"; import { join } from "node:path"; import { chalkError, chalkWarning, cliLink } from "../utilities/cliOutput.js"; import { createTempDir } from "../utilities/fileSystem.js"; -import { docs, getInTouch } from "../utilities/links.js"; +import { links } from "@trigger.dev/core/v3"; export type WarningsCheckReturn = | { @@ -46,27 +46,27 @@ export function checkLogsForErrors(logs: string) { const errors: LogParserOptions = [ { regex: /Error: Provided --schema at (?.*) doesn't exist/, - message: `Prisma generate failed to find the specified schema at "$schema".\nDid you include it in config.additionalFiles? ${cliLink( - "Config docs", - docs.config.prisma + message: `Prisma generate failed to find the specified schema at "$schema".\nDid you configure the Prisma extension correctly? ${cliLink( + "Extension docs", + links.docs.config.prisma )}`, }, { regex: /@prisma\/client did not initialize yet/, - message: `Prisma client not initialized yet.\nDid you forget to add the postinstall script? ${cliLink( - "Config docs", - docs.config.prisma + message: `Prisma client not initialized yet.\nDid you configure the Prisma extension? ${cliLink( + "Extension docs", + links.docs.config.prisma )}`, }, { regex: /sh: 1: (?.*): not found/, - message: `$packageOrBinary not found\n\nIf it's a package: Include it in ${cliLink( - "config.additionalPackages", - docs.config.prisma - )}\nIf it's a binary: Please ${cliLink( - "get in touch", - getInTouch - )} and we'll see what we can do!`, + message: `$packageOrBinary not found\n\nIf it's a package: Use the ${cliLink( + "additionalPackages extension", + links.docs.config.additionalPackages + )}\nIf it's a binary: Check the other ${cliLink( + "build extensions", + links.docs.config.extensions + )}`, }, ]; @@ -101,10 +101,10 @@ export function checkLogsForWarnings(logs: string): WarningsCheckReturn { const warnings: LogParserOptions = [ { regex: /prisma:warn We could not find your Prisma schema/, - message: `Prisma generate failed to find the default schema. Did you include it in config.additionalFiles? ${cliLink( - "Config docs", - docs.config.prisma - )}\nCustom schema paths require a postinstall script like this: \`prisma generate --schema=./custom/path/to/schema.prisma\``, + message: `Prisma generate failed to find the default schema. Did you configure the Prisma extension correctly? ${cliLink( + "Extension docs", + links.docs.config.prisma + )}`, shouldFail: true, }, ]; diff --git a/packages/cli-v3/src/dev/backgroundWorker.ts b/packages/cli-v3/src/dev/backgroundWorker.ts index 24ae8942f8..fcbc968f83 100644 --- a/packages/cli-v3/src/dev/backgroundWorker.ts +++ b/packages/cli-v3/src/dev/backgroundWorker.ts @@ -3,7 +3,6 @@ import { CreateBackgroundWorkerResponse, ServerBackgroundWorker, TaskRunBuiltInError, - TaskRunErrorCodes, TaskRunExecution, TaskRunExecutionPayload, TaskRunExecutionResult, @@ -14,13 +13,7 @@ import { import { Evt } from "evt"; import { join } from "node:path"; -import { - CancelledProcessError, - CleanupProcessError, - SigKillTimeoutProcessError, - UnexpectedExitError, - getFriendlyErrorMessage, -} from "@trigger.dev/core/v3/errors"; +import { SigKillTimeoutProcessError } from "@trigger.dev/core/v3/errors"; import { TaskRunProcess, TaskRunProcessOptions } from "../executions/taskRunProcess.js"; import { indexWorkerManifest } from "../indexing/indexWorkerManifest.js"; import { prettyError } from "../utilities/cliOutput.js"; @@ -490,7 +483,7 @@ export class BackgroundWorker { const error = result.error; if (error.type === "BUILT_IN_ERROR") { - const mappedError = await this.#correctError(error, payload.execution); + const mappedError = await this.#correctError(error); return { ...result, @@ -500,61 +493,16 @@ export class BackgroundWorker { return result; } catch (e) { - if (e instanceof CancelledProcessError) { - return { - id: payload.execution.run.id, - ok: false, - retry: undefined, - error: { - type: "INTERNAL_ERROR", - code: TaskRunErrorCodes.TASK_RUN_CANCELLED, - }, - }; - } - - if (e instanceof CleanupProcessError) { - return { - id: payload.execution.run.id, - ok: false, - retry: undefined, - error: { - type: "INTERNAL_ERROR", - code: TaskRunErrorCodes.TASK_EXECUTION_ABORTED, - }, - }; - } - - if (e instanceof UnexpectedExitError) { - return { - id: payload.execution.run.id, - ok: false, - retry: undefined, - error: { - type: "INTERNAL_ERROR", - code: TaskRunErrorCodes.TASK_PROCESS_EXITED_WITH_NON_ZERO_CODE, - message: getFriendlyErrorMessage(e.code, e.signal, e.stderr), - stackTrace: e.stderr, - }, - }; - } - return { id: payload.execution.run.id, ok: false, retry: undefined, - error: { - type: "INTERNAL_ERROR", - code: TaskRunErrorCodes.TASK_EXECUTION_FAILED, - message: String(e), - }, + error: TaskRunProcess.parseExecuteError(e), }; } } - async #correctError( - error: TaskRunBuiltInError, - execution: TaskRunExecution - ): Promise { + async #correctError(error: TaskRunBuiltInError): Promise { return { ...error, stackTrace: correctErrorStackTrace(error.stackTrace, this.params.cwd), diff --git a/packages/cli-v3/src/entryPoints/deploy-run-controller.ts b/packages/cli-v3/src/entryPoints/deploy-run-controller.ts index 8d0ae70ed2..8b3bc88611 100644 --- a/packages/cli-v3/src/entryPoints/deploy-run-controller.ts +++ b/packages/cli-v3/src/entryPoints/deploy-run-controller.ts @@ -784,7 +784,22 @@ class ProdWorker { error, }); - this.#failRun(message.lazyPayload.runId, error); + try { + await this._taskRunProcess.cancel(); + } catch (error) { + logger.error("Failed to cancel task run process", { error }); + } + + try { + await this.#submitAttemptCompletion(execution, { + id: execution.run.id, + ok: false, + retry: undefined, + error: TaskRunProcess.parseExecuteError(error, !this.runningInKubernetes), + }); + } catch (error) { + this.#failRun(message.lazyPayload.runId, error); + } } }, REQUEST_ATTEMPT_CANCELLATION: async (message) => { diff --git a/packages/cli-v3/src/executions/taskRunProcess.ts b/packages/cli-v3/src/executions/taskRunProcess.ts index bdeba68324..c7663c4ac5 100644 --- a/packages/cli-v3/src/executions/taskRunProcess.ts +++ b/packages/cli-v3/src/executions/taskRunProcess.ts @@ -1,9 +1,11 @@ import { ExecutorToWorkerMessageCatalog, ServerBackgroundWorker, + TaskRunErrorCodes, TaskRunExecution, TaskRunExecutionPayload, TaskRunExecutionResult, + type TaskRunInternalError, WorkerManifest, WorkerToExecutorMessageCatalog, } from "@trigger.dev/core/v3"; @@ -21,10 +23,10 @@ import { logger } from "../utilities/logger.js"; import { CancelledProcessError, CleanupProcessError, + internalErrorFromUnexpectedExit, GracefulExitTimeoutError, UnexpectedExitError, } from "@trigger.dev/core/v3/errors"; -import { env } from "std-env"; export type OnWaitForDurationMessage = InferSocketMessageSchema< typeof ExecutorToWorkerMessageCatalog, @@ -383,6 +385,39 @@ export class TaskRunProcess { get pid() { return this._childPid; } + + static parseExecuteError(error: unknown, dockerMode = true): TaskRunInternalError { + if (error instanceof CancelledProcessError) { + return { + type: "INTERNAL_ERROR", + code: TaskRunErrorCodes.TASK_RUN_CANCELLED, + }; + } + + if (error instanceof CleanupProcessError) { + return { + type: "INTERNAL_ERROR", + code: TaskRunErrorCodes.TASK_EXECUTION_ABORTED, + }; + } + + if (error instanceof UnexpectedExitError) { + return internalErrorFromUnexpectedExit(error, dockerMode); + } + + if (error instanceof GracefulExitTimeoutError) { + return { + type: "INTERNAL_ERROR", + code: TaskRunErrorCodes.GRACEFUL_EXIT_TIMEOUT, + }; + } + + return { + type: "INTERNAL_ERROR", + code: TaskRunErrorCodes.TASK_EXECUTION_FAILED, + message: String(error), + }; + } } function executorArgs(workerManifest: WorkerManifest): string[] { diff --git a/packages/cli-v3/src/utilities/deployErrors.ts b/packages/cli-v3/src/utilities/deployErrors.ts deleted file mode 100644 index 1fdb1e6f3a..0000000000 --- a/packages/cli-v3/src/utilities/deployErrors.ts +++ /dev/null @@ -1,201 +0,0 @@ -import chalk from "chalk"; -import { relative } from "node:path"; -import { - chalkError, - chalkPurple, - chalkGrey, - chalkGreen, - chalkWarning, - cliLink, -} from "./cliOutput.js"; -import { logger } from "./logger.js"; -import { z } from "zod"; -import { groupTaskMetadataIssuesByTask } from "@trigger.dev/core/v3"; -import { docs } from "./links.js"; - -export type ESMRequireError = { - type: "esm-require-error"; - moduleName: string; -}; - -export type BuildError = ESMRequireError | string; - -function errorIsErrorLike(error: unknown): error is Error { - return ( - error instanceof Error || (typeof error === "object" && error !== null && "message" in error) - ); -} - -export function parseBuildErrorStack(error: unknown): BuildError | undefined { - if (typeof error === "string") { - return error; - } - - if (errorIsErrorLike(error)) { - if (typeof error.stack === "string") { - if (error.stack.includes("ERR_REQUIRE_ESM")) { - const moduleName = getPackageNameFromEsmRequireError(error.stack); - - if (moduleName) { - return { - type: "esm-require-error", - moduleName, - }; - } - } - } else { - return error.message; - } - } - - return; -} - -function getPackageNameFromEsmRequireError(stack: string): string | undefined { - const pathRegex = /require\(\) of ES Module (.*) from/; - const pathMatch = pathRegex.exec(stack); - - if (!pathMatch) { - return; - } - - const filePath = pathMatch[1]; - - if (!filePath) { - return; - } - - const lastPart = filePath.split("node_modules/").pop(); - - if (!lastPart) { - return; - } - - // regular expression to match the package name - const moduleRegex = /(@[^\/]+\/[^\/]+|[^\/]+)/; - - const match = moduleRegex.exec(lastPart); - - if (!match) { - return; - } - - return match[1]; -} - -export function logESMRequireError(parsedError: ESMRequireError) { - logger.log( - `\n${chalkError("X Error:")} The ${chalkPurple( - parsedError.moduleName - )} module is being required even though it's ESM only, and builds only support CommonJS. There are two ${chalk.underline( - "possible" - )} ways to fix this:` - ); - logger.log( - `\n${chalkGrey("○")} Dynamically import the module in your code: ${chalkGrey( - `const myModule = await import("${parsedError.moduleName}");` - )}` - ); - - logger.log( - `${chalkGrey("○")} ${chalk.underline("Or")} add ${chalkPurple( - parsedError.moduleName - )} to the ${chalkGreen("dependenciesToBundle")} array in your config file ${chalkGrey( - "(you'll need to create one)" - )}. This will bundle the module with your code.\n` - ); - - logger.log( - `${chalkGrey("○")} For more info see the ${cliLink("relevant docs", docs.config.esm)}.\n` - ); -} - -export type PackageNotFoundError = { - type: "package-not-found-error"; - packageName: string; -}; - -export type NoMatchingVersionError = { - type: "no-matching-version-error"; - packageName: string; -}; - -export type NpmInstallError = PackageNotFoundError | NoMatchingVersionError | string; - -export function parseNpmInstallError(error: unknown): NpmInstallError { - if (typeof error === "string") { - return error; - } - - if (error instanceof Error) { - if (typeof error.stack === "string") { - const isPackageNotFoundError = - error.stack.includes("ERR! 404 Not Found") && - error.stack.includes("is not in this registry"); - let packageName = null; - - if (isPackageNotFoundError) { - // Regular expression to match the package name - const packageNameRegex = /'([^']+)' is not in this registry/; - const match = packageNameRegex.exec(error.stack); - if (match) { - packageName = match[1] as string; // Capture the package name - } - } - - if (packageName) { - return { - type: "package-not-found-error", - packageName, - }; - } - - const noMatchingVersionRegex = /No matching version found for ([^\s]+)\s/; - const noMatchingVersionMatch = noMatchingVersionRegex.exec(error.stack); - if (noMatchingVersionMatch) { - return { - type: "no-matching-version-error", - packageName: (noMatchingVersionMatch[1] as string).replace(/.$/, ""), - }; - } - - return error.message; - } else { - return error.message; - } - } - - return "Unknown error"; -} - -export function logTaskMetadataParseError(zodIssues: z.ZodIssue[], tasks: any) { - logger.log( - `\n${chalkError("X Error:")} Failed to start. The following ${ - zodIssues.length === 1 ? "task issue was" : "task issues were" - } found:` - ); - - const groupedIssues = groupTaskMetadataIssuesByTask(tasks, zodIssues); - - for (const key in groupedIssues) { - const taskWithIssues = groupedIssues[key]; - - if (!taskWithIssues) { - continue; - } - - logger.log( - `\n ${chalkWarning("❯")} ${taskWithIssues.exportName} ${chalkGrey("in")} ${ - taskWithIssues.filePath - }` - ); - - for (const issue of taskWithIssues.issues) { - if (issue.path) { - logger.log(` ${chalkError("x")} ${issue.path} ${chalkGrey(issue.message)}`); - } else { - logger.log(` ${chalkError("x")} ${chalkGrey(issue.message)}`); - } - } - } -} diff --git a/packages/cli-v3/src/utilities/links.ts b/packages/cli-v3/src/utilities/links.ts deleted file mode 100644 index e6b485ca76..0000000000 --- a/packages/cli-v3/src/utilities/links.ts +++ /dev/null @@ -1,10 +0,0 @@ -export const docs = { - config: { - home: "https://trigger.dev/docs/v3/trigger-config", - esm: "https://trigger.dev/docs/v3/trigger-config#esm-only-packages", - prisma: "https://trigger.dev/docs/v3/trigger-config#prisma-and-other-generators", - additionalPackages: "https://trigger.dev/docs/v3/trigger-config#prisma-and-other-generators", - }, -}; - -export const getInTouch = "https://trigger.dev/contact"; diff --git a/packages/core/src/v3/errors.ts b/packages/core/src/v3/errors.ts index 5feea6d6e7..17ae534149 100644 --- a/packages/core/src/v3/errors.ts +++ b/packages/core/src/v3/errors.ts @@ -1,8 +1,15 @@ import { z } from "zod"; import { DeploymentErrorData } from "./schemas/api.js"; import { ImportTaskFileErrors, WorkerManifest } from "./schemas/build.js"; -import { SerializedError, TaskRunError } from "./schemas/common.js"; +import { + SerializedError, + TaskRunError, + TaskRunErrorCodes, + TaskRunInternalError, +} from "./schemas/common.js"; import { TaskMetadataFailedToParseData } from "./schemas/messages.js"; +import { links } from "./links.js"; +import { ExceptionEventProperties } from "./schemas/openTelemetry.js"; export class AbortTaskRunError extends Error { constructor(message: string) { @@ -68,27 +75,29 @@ export function createErrorTaskError(error: TaskRunError): any { } export function createJsonErrorObject(error: TaskRunError): SerializedError { - switch (error.type) { + const enhancedError = taskRunErrorEnhancer(error); + + switch (enhancedError.type) { case "BUILT_IN_ERROR": { return { - name: error.name, - message: error.message, - stackTrace: error.stackTrace, + name: enhancedError.name, + message: enhancedError.message, + stackTrace: enhancedError.stackTrace, }; } case "STRING_ERROR": { return { - message: error.raw, + message: enhancedError.raw, }; } case "CUSTOM_ERROR": { return { - message: error.raw, + message: enhancedError.raw, }; } case "INTERNAL_ERROR": { return { - message: `trigger.dev internal error (${error.code})`, + message: `trigger.dev internal error (${enhancedError.code})`, }; } } @@ -265,7 +274,7 @@ export class UnexpectedExitError extends Error { public signal: NodeJS.Signals | null, public stderr: string | undefined ) { - super(`Unexpected exit with code ${code}`); + super(`Unexpected exit with code ${code} after signal ${signal}`); this.name = "UnexpectedExitError"; } @@ -303,40 +312,125 @@ export class GracefulExitTimeoutError extends Error { } } -export function getFriendlyErrorMessage( - code: number, - signal: NodeJS.Signals | null, - stderr: string | undefined, - dockerMode = true -) { - const message = (text: string) => { - if (signal) { - return `[${signal}] ${text}`; - } else { - return text; +const prettyInternalErrors: Partial< + Record +> = { + TASK_PROCESS_OOM_KILLED: { + message: + "Your task ran out of memory. Try increasing the machine specs. If this doesn't fix it there might be a memory leak.", + link: { + name: "Machines", + href: links.docs.machines.home, + }, + }, + TASK_PROCESS_MAYBE_OOM_KILLED: { + message: + "We think your task ran out of memory, but we can't be certain. If this keeps happening, try increasing the machine specs.", + link: { + name: "Machines", + href: links.docs.machines.home, + }, + }, +}; + +type EnhanceError = T & { + link?: { name: string; href: string }; +}; + +export function taskRunErrorEnhancer(error: TaskRunError): EnhanceError { + switch (error.type) { + case "BUILT_IN_ERROR": { + if (error.name === "UnexpectedExitError") { + if (error.message.startsWith("Unexpected exit with code -1")) { + return { + type: "INTERNAL_ERROR", + code: TaskRunErrorCodes.TASK_PROCESS_MAYBE_OOM_KILLED, + ...prettyInternalErrors.TASK_PROCESS_MAYBE_OOM_KILLED, + }; + } + } + break; } - }; + case "STRING_ERROR": { + break; + } + case "CUSTOM_ERROR": { + break; + } + case "INTERNAL_ERROR": { + if (error.code === TaskRunErrorCodes.TASK_PROCESS_EXITED_WITH_NON_ZERO_CODE) { + return { + type: "INTERNAL_ERROR", + code: TaskRunErrorCodes.TASK_PROCESS_MAYBE_OOM_KILLED, + ...prettyInternalErrors.TASK_PROCESS_MAYBE_OOM_KILLED, + }; + } + + const prettyError = prettyInternalErrors[error.code]; + + if (prettyError) { + return { + ...error, + ...prettyError, + }; + } + + break; + } + } - if (code === 137) { + return error; +} + +export function exceptionEventEnhancer( + exception: ExceptionEventProperties +): EnhanceError { + switch (exception.type) { + case "UnexpectedExitError": { + if (exception.message?.startsWith("Unexpected exit with code -1")) { + return { + ...exception, + ...prettyInternalErrors.TASK_PROCESS_MAYBE_OOM_KILLED, + }; + } + break; + } + } + + return exception; +} + +export function internalErrorFromUnexpectedExit( + error: UnexpectedExitError, + dockerMode = true +): TaskRunInternalError { + if (error.code === 137) { if (dockerMode) { - return message( - "Process ran out of memory! Try choosing a machine preset with more memory for this task." - ); + return { + type: "INTERNAL_ERROR", + code: TaskRunErrorCodes.TASK_PROCESS_OOM_KILLED, + }; } else { - // Note: containerState reason and message should be checked to clarify the error - return message( - "Process most likely ran out of memory, but we can't be certain. Try choosing a machine preset with more memory for this task." - ); + // Note: containerState reason and message could be checked to clarify the error, maybe the task monitor should be allowed to override these + return { + type: "INTERNAL_ERROR", + code: TaskRunErrorCodes.TASK_PROCESS_MAYBE_OOM_KILLED, + }; } } - if (stderr?.includes("OOMErrorHandler")) { - return message( - "Process ran out of memory! Try choosing a machine preset with more memory for this task." - ); + if (error.stderr?.includes("OOMErrorHandler")) { + return { + type: "INTERNAL_ERROR", + code: TaskRunErrorCodes.TASK_PROCESS_OOM_KILLED, + }; } - return message(`Process exited with code ${code}.`); + return { + type: "INTERNAL_ERROR", + code: TaskRunErrorCodes.TASK_PROCESS_EXITED_WITH_NON_ZERO_CODE, + message: `Process exited with code ${error.code} after signal ${error.signal}.`, + }; } export function serializeIndexingError(error: unknown, stderr?: string): DeploymentErrorData { diff --git a/packages/core/src/v3/index.ts b/packages/core/src/v3/index.ts index 3302028d8c..388e66d539 100644 --- a/packages/core/src/v3/index.ts +++ b/packages/core/src/v3/index.ts @@ -17,6 +17,7 @@ export * from "./schemas/index.js"; export { SemanticInternalAttributes } from "./semanticInternalAttributes.js"; export * from "./task-catalog-api.js"; export * from "./types/index.js"; +export { links } from "./links.js"; export { formatDuration, formatDurationInDays, diff --git a/packages/core/src/v3/links.ts b/packages/core/src/v3/links.ts new file mode 100644 index 0000000000..5f4f27210d --- /dev/null +++ b/packages/core/src/v3/links.ts @@ -0,0 +1,17 @@ +export const links = { + docs: { + config: { + home: "https://trigger.dev/docs/config/config-file", + additionalPackages: "https://trigger.dev/docs/config/config-file#additionalpackages", + extensions: "https://trigger.dev/docs/config/config-file#extensions", + prisma: "https://trigger.dev/docs/config/config-file#prisma", + }, + machines: { + home: "https://trigger.dev/docs/v3/machines", + }, + }, + site: { + home: "https://trigger.dev", + contact: "https://trigger.dev/contact", + }, +}; diff --git a/packages/core/src/v3/schemas/common.ts b/packages/core/src/v3/schemas/common.ts index cd4d3584ea..c7e2bd8f77 100644 --- a/packages/core/src/v3/schemas/common.ts +++ b/packages/core/src/v3/schemas/common.ts @@ -87,12 +87,17 @@ export const TaskRunErrorCodes = { TASK_EXECUTION_ABORTED: "TASK_EXECUTION_ABORTED", TASK_PROCESS_EXITED_WITH_NON_ZERO_CODE: "TASK_PROCESS_EXITED_WITH_NON_ZERO_CODE", TASK_PROCESS_SIGKILL_TIMEOUT: "TASK_PROCESS_SIGKILL_TIMEOUT", + TASK_PROCESS_OOM_KILLED: "TASK_PROCESS_OOM_KILLED", + TASK_PROCESS_MAYBE_OOM_KILLED: "TASK_PROCESS_MAYBE_OOM_KILLED", TASK_RUN_CANCELLED: "TASK_RUN_CANCELLED", TASK_OUTPUT_ERROR: "TASK_OUTPUT_ERROR", HANDLE_ERROR_ERROR: "HANDLE_ERROR_ERROR", GRACEFUL_EXIT_TIMEOUT: "GRACEFUL_EXIT_TIMEOUT", TASK_RUN_CRASHED: "TASK_RUN_CRASHED", MAX_DURATION_EXCEEDED: "MAX_DURATION_EXCEEDED", + DISK_SPACE_EXCEEDED: "DISK_SPACE_EXCEEDED", + POD_EVICTED: "POD_EVICTED", + POD_UNKNOWN_ERROR: "POD_UNKNOWN_ERROR", } as const; export const TaskRunInternalError = z.object({ @@ -107,6 +112,8 @@ export const TaskRunInternalError = z.object({ "TASK_EXECUTION_ABORTED", "TASK_PROCESS_EXITED_WITH_NON_ZERO_CODE", "TASK_PROCESS_SIGKILL_TIMEOUT", + "TASK_PROCESS_OOM_KILLED", + "TASK_PROCESS_MAYBE_OOM_KILLED", "TASK_RUN_CANCELLED", "TASK_OUTPUT_ERROR", "HANDLE_ERROR_ERROR", @@ -114,6 +121,9 @@ export const TaskRunInternalError = z.object({ "TASK_RUN_HEARTBEAT_TIMEOUT", "TASK_RUN_CRASHED", "MAX_DURATION_EXCEEDED", + "DISK_SPACE_EXCEEDED", + "POD_EVICTED", + "POD_UNKNOWN_ERROR", ]), message: z.string().optional(), stackTrace: z.string().optional(), diff --git a/packages/core/src/v3/schemas/messages.ts b/packages/core/src/v3/schemas/messages.ts index 746848640e..e35c7cf50a 100644 --- a/packages/core/src/v3/schemas/messages.ts +++ b/packages/core/src/v3/schemas/messages.ts @@ -5,6 +5,7 @@ import { TaskRunExecution, TaskRunExecutionResult, TaskRunFailedExecutionResult, + TaskRunInternalError, } from "./common.js"; import { TaskResource } from "./resources.js"; import { @@ -252,6 +253,7 @@ export const ProviderToPlatformMessages = { message: z.string().optional(), logs: z.string().optional(), overrideCompletion: z.boolean().optional(), + errorCode: TaskRunInternalError.shape.code.optional(), }), }, INDEXING_FAILED: {