feat: move rrweb event stream to client and query through /api/clickhouse-proxy

.changeset/neat-badgers-matter.md

@@ -0,0 +1,7 @@
+---
+"@hyperdx/common-utils": patch
+"@hyperdx/api": patch
+"@hyperdx/app": patch
+---
+
+feat: move rrweb event fetching to the client instead of an api route

packages/api/src/api-app.ts

@@ -91,7 +91,6 @@ app.use('/dashboards', isUserAuthenticated, routers.dashboardRouter);
 app.use('/logs', isUserAuthenticated, routers.logsRouter);
 app.use('/me', isUserAuthenticated, routers.meRouter);
 app.use('/metrics', isUserAuthenticated, routers.metricsRouter);
-app.use('/sessions', isUserAuthenticated, routers.sessionsRouter);
 app.use('/team', isUserAuthenticated, routers.teamRouter);
 app.use('/webhooks', isUserAuthenticated, routers.webhooksRouter);
 app.use('/chart', isUserAuthenticated, routers.chartRouter);

packages/api/src/routers/api/clickhouseProxy.ts

@@ -1,10 +1,11 @@
-import express, { Request, Response } from 'express';
+import express, { RequestHandler, Response } from 'express';
 import { createProxyMiddleware } from 'http-proxy-middleware';
 import { z } from 'zod';
 import { validateRequest } from 'zod-express-middleware';
 
 import { getConnectionById } from '@/controllers/connection';
 import { getNonNullUserWithTeam } from '@/middleware/auth';
+import { validateRequestHeaders } from '@/utils/validation';
 import { objectIdSchema } from '@/utils/zod';
 
 const router = express.Router();
@@ -58,17 +59,23 @@ router.post(
   },
 );
 
-router.get(
-  '/*',
-  validateRequest({
-    query: z.object({
-      hyperdx_connection_id: objectIdSchema,
+function validation() {
+  return validateRequestHeaders(
+    z.object({
+      'x-hyperdx-connection-id': objectIdSchema,
     }),
-  }),
-  async (req, res, next) => {
+  );
+}
+
+function getConnection(): RequestHandler {
+  return async (req, res, next) => {
     try {
       const { teamId } = getNonNullUserWithTeam(req);
-      const { hyperdx_connection_id } = req.query;
+      const connection_id = req.headers['x-hyperdx-connection-id']!; // ! because zod already validated
+      delete req.headers['x-hyperdx-connection-id'];
+      const hyperdx_connection_id = Array.isArray(connection_id)
+        ? connection_id.join('')
+        : connection_id;
 
       const connection = await getConnectionById(
         teamId.toString(),
@@ -93,13 +100,15 @@ router.get(
       console.error('Error fetching connection info:', e);
       next(e);
     }
-  },
-  createProxyMiddleware({
+  };
+}
+
+function proxyMiddleware(): RequestHandler {
+  return createProxyMiddleware({
     target: '', // doesn't matter. it should be overridden by the router
     changeOrigin: true,
     pathFilter: (path, _req) => {
-      // TODO: allow other methods
-      return _req.method === 'GET';
+      return _req.method === 'GET' || _req.method === 'POST';
     },
     pathRewrite: {
       '^/clickhouse-proxy': '',
@@ -113,16 +122,23 @@ router.get(
     on: {
       proxyReq: (proxyReq, _req) => {
         const newPath = _req.params[0];
-        const qparams = new URLSearchParams(_req.query);
-        qparams.delete('hyperdx_connection_id');
+        let qparams = '';
+        const qIdx = _req.url.indexOf('?');
+        if (qIdx >= 0) {
+          qparams = _req.url.substring(qIdx);
+        }
         if (_req._hdx_connection?.username && _req._hdx_connection?.password) {
           proxyReq.setHeader(
             'X-ClickHouse-User',
             _req._hdx_connection.username,
           );
           proxyReq.setHeader('X-ClickHouse-Key', _req._hdx_connection.password);
         }
-        proxyReq.path = `/${newPath}?${qparams.toString()}`;
+        if (_req.method === 'POST') {
+          // TODO: Use fixRequestBody after this issue is resolved: https://github.com/chimurai/http-proxy-middleware/issues/1102
+          proxyReq.write(_req.body);
+        }
+        proxyReq.path = `/${newPath}${qparams}`;
       },
       proxyRes: (proxyRes, _req, res) => {
         // since clickhouse v24, the cors headers * will be attached to the response by default
@@ -158,7 +174,10 @@ router.get(
     // ...(config.IS_DEV && {
     //   logger: console,
     // }),
-  }),
-);
+  });
+}
+
+router.get('/*', validation(), getConnection(), proxyMiddleware());
+router.post('/*', validation(), getConnection(), proxyMiddleware());
 
 export default router;

packages/api/src/routers/api/index.ts

@@ -6,7 +6,6 @@ import logsRouter from './logs';
 import meRouter from './me';
 import metricsRouter from './metrics';
 import rootRouter from './root';
-import sessionsRouter from './sessions';
 import teamRouter from './team';
 import webhooksRouter from './webhooks';
 
@@ -18,7 +17,6 @@ export default {
   meRouter,
   metricsRouter,
   rootRouter,
-  sessionsRouter,
   teamRouter,
   webhooksRouter,
   chartRouter,

packages/api/src/utils/validation.ts

@@ -0,0 +1,17 @@
+import express from 'express';
+import { z } from 'zod';
+
+export function validateRequestHeaders<T extends z.Schema>(schema: T) {
+  return function (
+    req: express.Request,
+    res: express.Response,
+    next: express.NextFunction,
+  ) {
+    const parsed = schema.safeParse(req.headers);
+    if (!parsed.success) {
+      return res.status(400).json({ type: 'Headers', errors: parsed.error });
+    }
+
+    return next();
+  };
+}

packages/app/pages/api/[...all].ts

@@ -6,7 +6,8 @@ const DEFAULT_SERVER_URL = `http://127.0.0.1:${process.env.HYPERDX_API_PORT}`;
 export const config = {
   api: {
     externalResolver: true,
-    bodyParser: true,
+    bodyParser: false,
+    responseLimit: '32mb',
   },
 };
 
@@ -17,12 +18,6 @@ export default (req: NextApiRequest, res: NextApiResponse) => {
     pathRewrite: { '^/api': '' },
     target: process.env.NEXT_PUBLIC_SERVER_URL || DEFAULT_SERVER_URL,
     autoRewrite: true,
-    /**
-     * Fix bodyParser
-     **/
-    on: {
-      proxyReq: fixRequestBody,
-    },
     // ...(IS_DEV && {
     //   logger: console,
     // }),