Merge pull request #2119 from dubinc/fix-crawl

Fix crawl by moving to function

Author: steven-tey

apps/web/app/api/links/crawl/bitly/route.ts

@@ -1,16 +1,21 @@
+import { createId } from "@/lib/api/create-id";
+import { ExpandedLink } from "@/lib/api/links";
+import { conn } from "@/lib/planetscale";
+import { recordLink } from "@/lib/tinybird/record-link";
 import { redis } from "@/lib/upstash";
+import { getUrlFromStringIfValid, linkConstructorSimple } from "@dub/utils";
+import { waitUntil } from "@vercel/functions";
 import { NextResponse } from "next/server";
 import { z } from "zod";
 
 // POST /api/links/crawl/bitly – crawl a bitly link
 export const POST = async (req: Request) => {
   const body = await req.json();
 
-  const { domain, key, workspaceId } = z
+  const { domain, key } = z
     .object({
       domain: z.string(),
       key: z.string(),
-      workspaceId: z.string(),
     })
     .parse(body);
 
@@ -21,11 +26,74 @@ export const POST = async (req: Request) => {
     return NextResponse.json({ error: "Not found" }, { status: 404 });
   }
 
+  const workspaceId = "cm05wnnpo000711ztj05wwdbu";
+  const userId = "cm05wnd49000411ztg2xbup0i";
+  const folderId = "fold_LIZsdjTgFVbQVGYSUmYAi5vT";
+
   const link = await crawlBitlyLink({ domain, key, workspaceId });
 
-  return link
-    ? NextResponse.json(link)
-    : NextResponse.json({ error: "Not found" }, { status: 404 });
+  if (!link) {
+    return NextResponse.json({ error: "Not found" }, { status: 404 });
+  }
+
+  const sanitizedUrl = getUrlFromStringIfValid(link.long_url);
+
+  if (!sanitizedUrl) {
+    return NextResponse.json({ error: "Not found" }, { status: 404 });
+  }
+
+  const newLink = {
+    id: createId({ prefix: "link_" }),
+    projectId: workspaceId,
+    userId,
+    domain,
+    key,
+    url: sanitizedUrl,
+    shortLink: linkConstructorSimple({
+      domain,
+      key,
+    }),
+    archived: false,
+    folderId,
+    createdAt: new Date(link.created_at),
+    updatedAt: new Date(link.created_at),
+  };
+
+  console.log("[Bitly] Creating link", newLink);
+
+  try {
+    await conn.execute(
+      "INSERT INTO Link (id, projectId, userId, domain, `key`, url, shortLink, archived, folderId, createdAt, updatedAt) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+      [
+        newLink.id,
+        newLink.projectId,
+        newLink.userId,
+        newLink.domain,
+        newLink.key,
+        newLink.url,
+        newLink.shortLink,
+        newLink.archived,
+        newLink.folderId,
+        newLink.createdAt,
+        newLink.updatedAt,
+      ],
+    );
+
+    waitUntil(
+      recordLink({
+        ...newLink,
+        tenantId: null,
+        programId: null,
+        partnerId: null,
+        tags: [],
+      } as unknown as ExpandedLink),
+    );
+
+    return NextResponse.json(newLink);
+  } catch (error) {
+    console.error("[Bitly] Error creating link", error);
+    return NextResponse.json({ error: "Not found" }, { status: 404 });
+  }
 };
 
 async function crawlBitlyLink({

apps/web/lib/middleware/bitly.ts

@@ -1,22 +1,5 @@
-import {
-  APP_DOMAIN,
-  getUrlFromStringIfValid,
-  linkConstructorSimple,
-} from "@dub/utils";
-import { waitUntil } from "@vercel/functions";
-import { z } from "zod";
-import { ExpandedLink } from "../api/links";
-
-import { createId } from "../api/create-id";
+import { APP_DOMAIN } from "@dub/utils";
 import { EdgeLinkProps } from "../planetscale";
-import { conn } from "../planetscale/connection";
-import { recordLink } from "../tinybird/record-link";
-
-const crawlBitlyResultSchema = z.object({
-  id: z.string(),
-  long_url: z.string(),
-  created_at: z.string(),
-});
 
 // Create a new Bitly-hosted link to Dub on-demand (e.g. buff.ly)
 export const importBitlyLink = async ({
@@ -26,89 +9,19 @@ export const importBitlyLink = async ({
   domain: string;
   key: string;
 }) => {
-  const workspaceId = "cm05wnnpo000711ztj05wwdbu";
-  const userId = "cm05wnd49000411ztg2xbup0i";
-  const folderId = "fold_LIZsdjTgFVbQVGYSUmYAi5vT";
-
-  let link: z.infer<typeof crawlBitlyResultSchema> | null = null;
-
   try {
     const result = await fetch(`${APP_DOMAIN}/api/links/crawl/bitly`, {
       method: "POST",
-      body: JSON.stringify({ domain, key, workspaceId }),
+      body: JSON.stringify({ domain, key }),
     });
 
     if (result.status === 404) {
       return null;
     }
 
-    link = crawlBitlyResultSchema.parse(await result.json());
+    return (await result.json()) as EdgeLinkProps;
   } catch (e) {
     console.error("[Bitly] Error crawling Bitly link", e);
     return null;
   }
-
-  if (!link) {
-    return null;
-  }
-
-  const sanitizedUrl = getUrlFromStringIfValid(link.long_url);
-
-  if (!sanitizedUrl) {
-    return null;
-  }
-
-  const newLink = {
-    id: createId({ prefix: "link_" }),
-    projectId: workspaceId,
-    userId,
-    domain,
-    key,
-    url: sanitizedUrl,
-    shortLink: linkConstructorSimple({
-      domain,
-      key,
-    }),
-    archived: false,
-    folderId,
-    createdAt: new Date(link.created_at),
-    updatedAt: new Date(link.created_at),
-  };
-
-  console.log("[Bitly] Creating link", newLink);
-
-  try {
-    await conn.execute(
-      "INSERT INTO Link (id, projectId, userId, domain, `key`, url, shortLink, archived, folderId, createdAt, updatedAt) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
-      [
-        newLink.id,
-        newLink.projectId,
-        newLink.userId,
-        newLink.domain,
-        newLink.key,
-        newLink.url,
-        newLink.shortLink,
-        newLink.archived,
-        newLink.folderId,
-        newLink.createdAt,
-        newLink.updatedAt,
-      ],
-    );
-
-    // TODO: fetch tags
-    waitUntil(
-      recordLink({
-        ...newLink,
-        tenantId: null,
-        programId: null,
-        partnerId: null,
-        tags: [],
-      } as unknown as ExpandedLink),
-    );
-
-    return newLink as unknown as EdgeLinkProps;
-  } catch (error) {
-    console.error("[Bitly] Error creating link", error);
-    return null;
-  }
 };