feat(backend,shared,clerk-js): Support suffixed cookies (#3506)

Co-authored-by: Nikos Douvlis <[email protected]>

Author: dimkl

.changeset/calm-readers-call.md

@@ -0,0 +1,8 @@
+---
+'@clerk/clerk-js': minor
+'@clerk/backend': minor
+'@clerk/shared': minor
+---
+
+Support reading / writing / removing  suffixed/un-suffixed cookies from `@clerk/clerk-js` and `@clerk/backend`.
+The `__session`, `__clerk_db_jwt` and `__client_uat` cookies will now include a suffix derived from the instance's publishakeKey. The cookie name suffixes are used to prevent cookie collisions, effectively enabling support for multiple Clerk applications running on the same domain.

.github/actions/init/action.yml

@@ -106,14 +106,33 @@ runs:
     - name: Setup NodeJS ${{ inputs.node-version }}
       uses: actions/setup-node@v4
       with:
-        cache: 'npm'
         node-version: ${{ inputs.node-version }}
         registry-url: ${{ inputs.registry-url }}
 
+    - name: NPM debug
+      shell: bash
+      run: npm config ls -l
+
+    - name: Restore node_modules
+      uses: actions/cache/restore@v4
+      id: cache-npm
+      with:
+        path: ./node_modules
+        key: ${{ runner.os }}-node-${{ inputs.node-version }}-node-modules-${{ hashFiles('**/package-lock.json') }}-v6
+        restore-keys: ${{ runner.os }}-node-${{ inputs.node-version }}-node-modules-
+
     - name: Install NPM Dependencies
-      run: mkdir node_modules && npm ci --cache /home/runner/.npm --audit=false --fund=false --prefer-offline
+      # if: steps.cache-npm.outputs.cache-hit != 'true'
+      run: npm ci --prefer-offline --audit=false --fund=false  --verbose
       shell: bash
 
+    - name: Cache node_modules
+      uses: actions/cache/save@v4
+      if: steps.cache-npm.outputs.cache-hit != 'true'
+      with:
+        path: ./node_modules
+        key: ${{ runner.os }}-node-${{ inputs.node-version }}-node-modules-${{ hashFiles('**/package-lock.json') }}-v6
+
     - name: Get Playwright Version
       if: inputs.playwright-enabled == 'true'
       shell: bash

.github/workflows/ci.yml

@@ -37,10 +37,6 @@ jobs:
           turbo-team: ${{ vars.TURBO_TEAM }}
           turbo-token: ${{ secrets.TURBO_TOKEN }}
 
-      - name: List node_modules
-        run: npm ls
-        shell: bash
-
       - name: Require Changeset
         if: ${{ !(github.event_name == 'merge_group') }}
         run: if [[ "${{ github.event.pull_request.user.login }}" = "clerk-cookie" || "${{ github.event.pull_request.user.login }}" = "renovate[bot]" ]]; then echo 'Skipping' && exit 0; else npx changeset status --since=origin/main; fi
@@ -140,7 +136,7 @@ jobs:
 
     strategy:
       matrix:
-        test-name: ['generic', 'express', 'quickstart', 'ap-flows', 'elements', 'astro']
+        test-name: ['generic', 'express', 'quickstart', 'ap-flows', 'elements', 'astro', 'sessions']
         test-project: ['chrome']
         include:
           - test-name: 'nextjs'
@@ -157,6 +153,15 @@ jobs:
           fetch-depth: 0
           show-progress: false
 
+      - name: Setup
+        id: config
+        uses: ./.github/actions/init
+        with:
+          turbo-signature: ${{ secrets.TURBO_REMOTE_CACHE_SIGNATURE_KEY }}
+          turbo-team: ${{ vars.TURBO_TEAM }}
+          turbo-token: ${{ secrets.TURBO_TOKEN }}
+          playwright-enabled: true
+
       - name: Task Status
         id: task-status
         env:
@@ -170,16 +175,6 @@ jobs:
           (npx turbo-ignore --task=test:integration:${{ matrix.test-name }} --fallback=${{ github.base_ref || 'refs/heads/main' }}) || AFFECTED=1
           echo "affected=${AFFECTED}" >> $GITHUB_OUTPUT
 
-      - name: Setup
-        if: ${{ steps.task-status.outputs.affected == '1' }}
-        id: config
-        uses: ./.github/actions/init
-        with:
-          turbo-signature: ${{ secrets.TURBO_REMOTE_CACHE_SIGNATURE_KEY }}
-          turbo-team: ${{ vars.TURBO_TEAM }}
-          turbo-token: ${{ secrets.TURBO_TOKEN }}
-          playwright-enabled: true
-
       - name: Verdaccio
         if: ${{ steps.task-status.outputs.affected == '1' }}
         uses: ./.github/actions/verdaccio
@@ -201,10 +196,33 @@ jobs:
         if: ${{ matrix.test-name == 'astro' }}
         run: cd packages/astro && npm run copy:components
 
+      - name: Write all ENV certificates to files in integration/certs
+        if: ${{ steps.task-status.outputs.affected == '1' }}
+        uses: actions/github-script@v7
+        env:
+          INTEGRATION_CERTS: '${{secrets.INTEGRATION_CERTS}}'
+          INTEGRATION_ROOT_CA: '${{secrets.INTEGRATION_ROOT_CA}}'
+        with:
+            script: |
+                const fs = require('fs');
+                const path = require('path');
+                const rootCa = process.env.INTEGRATION_ROOT_CA;
+                console.log('rootCa', rootCa);
+                fs.writeFileSync(path.join(process.env.GITHUB_WORKSPACE, 'integration/certs', 'rootCA.pem'), rootCa);
+                const certs = JSON.parse(process.env.INTEGRATION_CERTS);
+                for (const [name, cert] of Object.entries(certs)) {
+                    fs.writeFileSync(path.join(process.env.GITHUB_WORKSPACE, 'integration/certs', name), cert);
+                }
+
+      - name: LS certs
+        if: ${{ steps.task-status.outputs.affected == '1' }}
+        working-directory: ./integration/certs
+        run: ls -la && pwd
+
       - name: Run Integration Tests
         if: ${{ steps.task-status.outputs.affected == '1' }}
         id: integration-tests
-        run: npx turbo test:integration:${{ matrix.test-name }} $TURBO_ARGS --only -- --project=${{ matrix.test-project }}
+        run: sudo --preserve-env npx turbo test:integration:${{ matrix.test-name }} $TURBO_ARGS --only -- --project=${{ matrix.test-project }}
         env:
           E2E_APP_CLERK_JS_DIR: ${{runner.temp}}
           E2E_CLERK_VERSION: 'latest'
@@ -213,6 +231,8 @@ jobs:
           E2E_CLERK_ENCRYPTION_KEY: ${{ matrix.clerk-encryption-key }}
           INTEGRATION_INSTANCE_KEYS: ${{ secrets.INTEGRATION_INSTANCE_KEYS }}
           MAILSAC_API_KEY: ${{ secrets.MAILSAC_API_KEY }}
+          NODE_EXTRA_CA_CERTS: ${{ github.workspace }}/integration/certs/rootCA.pem
+
 
       - name: Upload test-results
         if: ${{ cancelled() || failure() }}

integration/README.md

@@ -577,6 +577,10 @@ This is why you created the `.keys.json` file in the [initial setup](#initial-se
 
 They keys defined in `.keys.json.sample` correspond with the Clerk instances in the **Integration testing** organization.
 
+### Test isolation
+
+Before writing tests, it's important to understand how Playwright handles test isolation. Refer to the [Playwright documentation](https://playwright.dev/docs/browser-contexts) for more details.
+
 > [!NOTE]
 > The test suite also uses these environment variables to run some tests:
 >

integration/certs/README.md

@@ -0,0 +1,44 @@
+### Introduction
+
+Some of our e2e test suites require self-signed SSL certificates to be installed on the local machine. This short guide will walk you through the process of generating self-signed SSL certificates using `mkcert`.
+
+### Prerequisites
+
+Good news! If you've set up your local development environment for Clerk, you've already installed `mkcert` as part of our `make deps` command. If you haven't, you can install it by following the instructions [here](https://github.com/FiloSottile/mkcert)
+
+### Generate SSL Certificates
+
+To generate a new cert/key pair, you can simply run the following command:
+
+```bash
+mkcert -cert-file example.pem -key-file example-key.pem "example.com" "*.example.com"
+```
+
+The command above will create a `example.pem` and a `example-key.pem` file in the current directory. The certificate will be valid for `example.com` and all subdomains of `example.com`.
+
+### Using the Certificates
+
+During installation, `mkcert` automatically adds its root CA to your machine's trust store. All certificates generated by `mkcert` from that point on, will you that specific root CA. This means that you can use the generated certificates in your local development environment without any additional configuration. There's an important caveat though: `node` does not use the system root store, so it won't accept mkcert certificates automatically. Instead, you will have to set the `NODE_EXTRA_CA_CERTS` environment variable.
+
+```shell
+export NODE_EXTRA_CA_CERTS="$(mkcert -CAROOT)/rootCA.pem"
+```
+
+or provide the `NODE_EXTRA_CA_CERTS` when runnning your tests:
+
+```shell
+NODE_EXTRA_CA_CERTS="$(mkcert -CAROOT)/rootCA.pem" playwright test...
+```
+
+For more details, see [here](https://github.com/FiloSottile/mkcert?tab=readme-ov-file#changing-the-location-of-the-ca-files)
+
+### Github actions
+
+In order to avoid install mkcert and generating self-signed certificates in our CI/CD pipeline, we have added the generated certificates and the root CA to the repository's secrets:
+
+```shell
+secrets.INTEGRATION_ROOT_CA
+secrets.INTEGRATION_CERTS
+```
+
+During the CICD run, the certificates are loaded from the ENV and written to the `ingration/certs` directory.

integration/constants.ts

@@ -4,6 +4,7 @@ import * as path from 'node:path';
 
 export const constants = {
   TMP_DIR: path.join(os.tmpdir(), '.temp_integration'),
+  CERTS_DIR: path.join(process.cwd(), 'integration/certs'),
   APPS_STATE_FILE: path.join(os.tmpdir(), '.temp_integration', 'state.json'),
   /**
    * A URL to a running app that will be used to run the tests against.

integration/models/application.ts

@@ -6,7 +6,12 @@ import type { EnvironmentConfig } from './environment.js';
 
 export type Application = ReturnType<typeof application>;
 
-export const application = (config: ApplicationConfig, appDirPath: string, appDirName: string) => {
+export const application = (
+  config: ApplicationConfig,
+  appDirPath: string,
+  appDirName: string,
+  serverUrl: string | undefined,
+) => {
   const { name, scripts, envWriter } = config;
   const logger = createLogger({ prefix: `${appDirName}` });
   const state = { completedSetup: false, serverUrl: '', env: {} as EnvironmentConfig };
@@ -39,18 +44,24 @@ export const application = (config: ApplicationConfig, appDirPath: string, appDi
         state.completedSetup = true;
       }
     },
-    dev: async (opts: { port?: number; manualStart?: boolean; detached?: boolean } = {}) => {
+    dev: async (opts: { port?: number; manualStart?: boolean; detached?: boolean; serverUrl?: string } = {}) => {
       const log = logger.child({ prefix: 'dev' }).info;
       const port = opts.port || (await getPort());
-      const serverUrl = `http://localhost:${port}`;
-      log(`Will try to serve app at ${serverUrl}`);
+      const getServerUrl = () => {
+        if (opts.serverUrl) {
+          return opts.serverUrl.includes(':') ? opts.serverUrl : `${opts.serverUrl}:${port}`;
+        }
+        return serverUrl || `http://localhost:${port}`;
+      };
+      const runtimeServerUrl = getServerUrl();
+      log(`Will try to serve app at ${runtimeServerUrl}`);
       if (opts.manualStart) {
         // for debugging, you can start the dev server manually by cd'ing into the temp dir
         // and running the corresponding dev command
         // this allows the test to run as normally, while setup is controlled by you,
         // so you can inspect the running up outside the PW lifecycle
-        state.serverUrl = serverUrl;
-        return { port, serverUrl };
+        state.serverUrl = runtimeServerUrl;
+        return { port, serverUrl: runtimeServerUrl };
       }
 
       const proc = run(scripts.dev, {
@@ -61,12 +72,13 @@ export const application = (config: ApplicationConfig, appDirPath: string, appDi
         stderr: opts.detached ? fs.openSync(stderrFilePath, 'a') : undefined,
         log: opts.detached ? undefined : log,
       });
+
       const shouldExit = () => !!proc.exitCode && proc.exitCode !== 0;
-      await waitForServer(serverUrl, { log, maxAttempts: Infinity, shouldExit });
-      log(`Server started at ${serverUrl}, pid: ${proc.pid}`);
+      await waitForServer(runtimeServerUrl, { log, maxAttempts: Infinity, shouldExit });
+      log(`Server started at ${runtimeServerUrl}, pid: ${proc.pid}`);
       cleanupFns.push(() => awaitableTreekill(proc.pid, 'SIGKILL'));
-      state.serverUrl = serverUrl;
-      return { port, serverUrl, pid: proc.pid };
+      state.serverUrl = runtimeServerUrl;
+      return { port, serverUrl: runtimeServerUrl, pid: proc.pid };
     },
     build: async () => {
       const log = logger.child({ prefix: 'build' }).info;
@@ -83,6 +95,7 @@ export const application = (config: ApplicationConfig, appDirPath: string, appDi
     },
     serve: async (opts: { port?: number; manualStart?: boolean } = {}) => {
       const port = opts.port || (await getPort());
+      // TODO: get serverUrl as in dev()
       const serverUrl = `http://localhost:${port}`;
       // If this is ever used as a background process, we need to make sure
       // it's not using the log function. See the dev() method above

integration/models/applicationConfig.ts

@@ -12,6 +12,7 @@ type Scripts = { dev: string; build: string; setup: string; serve: string };
 
 export const applicationConfig = () => {
   let name = '';
+  let serverUrl = '';
   const templates: string[] = [];
   const files = new Map<string, string>();
   const scripts: Scripts = { dev: 'npm run dev', serve: 'npm run serve', build: 'npm run build', setup: 'npm i' };
@@ -35,6 +36,10 @@ export const applicationConfig = () => {
       name = _name;
       return self;
     },
+    setServerUrl: (_serverUrl: string) => {
+      serverUrl = _serverUrl;
+      return self;
+    },
     addFile: (filePath: string, cbOrPath: (helpers: Helpers) => string) => {
       files.set(filePath, cbOrPath(helpers));
       return self;
@@ -96,7 +101,7 @@ export const applicationConfig = () => {
         await fs.writeJSON(packageJsonPath, contents, { spaces: 2 });
       }
 
-      return application(self, appDirPath, appDirName);
+      return application(self, appDirPath, appDirName, serverUrl);
     },
     setEnvWriter: () => {
       throw new Error('not implemented');
@@ -115,9 +120,15 @@ export const applicationConfig = () => {
         logger.info(`Creating env file ".env" -> ${envDest}`);
         await fs.writeFile(
           path.join(appDir, '.env'),
-          [...env.publicVariables].map(([k, v]) => `${envFormatters.public(k)}=${v}`).join('\n') +
+          [...env.publicVariables]
+            .filter(([_, v]) => v)
+            .map(([k, v]) => `${envFormatters.public(k)}=${v}`)
+            .join('\n') +
             '\n' +
-            [...env.privateVariables].map(([k, v]) => `${envFormatters.private(k)}=${v}`).join('\n'),
+            [...env.privateVariables]
+              .filter(([_, v]) => v)
+              .map(([k, v]) => `${envFormatters.private(k)}=${v}`)
+              .join('\n'),
         );
       };
       return defaultWriter;

integration/models/environment.ts

@@ -7,7 +7,6 @@ export type EnvironmentConfig = {
   get id(): string;
   setId(newId: string): EnvironmentConfig;
   setEnvVariable(type: keyof EnvironmentVariables, name: string, value: any): EnvironmentConfig;
-  removeEnvVariable(type: keyof EnvironmentVariables, name: string): EnvironmentConfig;
   get publicVariables(): EnvironmentVariables['public'];
   get privateVariables(): EnvironmentVariables['private'];
   toJson(): { public: Record<string, string>; private: Record<string, string> };
@@ -34,10 +33,6 @@ export const environmentConfig = () => {
       envVars[type].set(name, value);
       return self;
     },
-    removeEnvVariable: (type, name) => {
-      envVars[type].delete(name);
-      return self;
-    },
     get publicVariables() {
       return envVars.public;
     },

integration/playwright.config.ts

@@ -18,13 +18,15 @@ export const common: PlaywrightTestConfig = {
   workers: process.env.CI ? '50%' : '70%',
   reporter: process.env.CI ? 'line' : 'list',
   use: {
+    ignoreHTTPSErrors: true,
     trace: 'retain-on-failure',
     bypassCSP: true, // We probably need to limit this to specific tests
   },
 } as const;
 
 export default defineConfig({
   ...common,
+
   projects: [
     {
       name: 'setup',