feat: add e2e testing for llamacloud datasource (#181)

---------
Co-authored-by: Marcus Schiesser <[email protected]>

Author: thucpn

.changeset/great-spies-cover.md

@@ -0,0 +1,5 @@
+---
+"create-llama": patch
+---
+
+Add e2e testing for llamacloud datasource

.github/workflows/e2e.yml

@@ -62,6 +62,7 @@ jobs:
         run: pnpm run e2e
         env:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          LLAMA_CLOUD_API_KEY: ${{ secrets.LLAMA_CLOUD_API_KEY }}
         working-directory: .
 
       - uses: actions/upload-artifact@v3

e2e/basic.spec.ts

@@ -17,20 +17,27 @@ const templateFrameworks: TemplateFramework[] = [
   "express",
   "fastapi",
 ];
-const dataSources: string[] = ["--no-files", "--example-file"];
+const dataSources: string[] = ["--no-files", "--llamacloud"];
 const templateUIs: TemplateUI[] = ["shadcn", "html"];
 const templatePostInstallActions: TemplatePostInstallAction[] = [
   "none",
   "runApp",
 ];
 
+const llamaCloudProjectName = "create-llama";
+const llamaCloudIndexName = "e2e-test";
+
 for (const templateType of templateTypes) {
   for (const templateFramework of templateFrameworks) {
     for (const dataSource of dataSources) {
       for (const templateUI of templateUIs) {
         for (const templatePostInstallAction of templatePostInstallActions) {
           const appType: AppType =
             templateFramework === "nextjs" ? "" : "--frontend";
+          const userMessage =
+            dataSource !== "--no-files"
+              ? "Physical standard for letters"
+              : "Hello";
           test.describe(`try create-llama ${templateType} ${templateFramework} ${dataSource} ${templateUI} ${appType} ${templatePostInstallAction}`, async () => {
             let port: number;
             let externalPort: number;
@@ -55,6 +62,8 @@ for (const templateType of templateTypes) {
                 port,
                 externalPort,
                 templatePostInstallAction,
+                llamaCloudProjectName,
+                llamaCloudIndexName,
               );
               name = result.projectName;
               appProcess = result.appProcess;
@@ -75,7 +84,7 @@ for (const templateType of templateTypes) {
             }) => {
               test.skip(templatePostInstallAction !== "runApp");
               await page.goto(`http://localhost:${port}`);
-              await page.fill("form input", "hello");
+              await page.fill("form input", userMessage);
               const [response] = await Promise.all([
                 page.waitForResponse(
                   (res) => {
@@ -106,7 +115,7 @@ for (const templateType of templateTypes) {
                     messages: [
                       {
                         role: "user",
-                        content: "Hello",
+                        content: userMessage,
                       },
                     ],
                   },

e2e/utils.ts

@@ -72,9 +72,13 @@ export async function runCreateLlama(
   port: number,
   externalPort: number,
   postInstallAction: TemplatePostInstallAction,
+  llamaCloudProjectName: string,
+  llamaCloudIndexName: string,
 ): Promise<CreateLlamaResult> {
-  if (!process.env.OPENAI_API_KEY) {
-    throw new Error("Setting OPENAI_API_KEY is mandatory to run tests");
+  if (!process.env.OPENAI_API_KEY || !process.env.LLAMA_CLOUD_API_KEY) {
+    throw new Error(
+      "Setting the OPENAI_API_KEY and LLAMA_CLOUD_API_KEY is mandatory to run tests",
+    );
   }
   const name = [
     templateType,
@@ -110,12 +114,16 @@ export async function runCreateLlama(
     "--no-llama-parse",
     "--observability",
     "none",
+    "--llama-cloud-key",
+    process.env.LLAMA_CLOUD_API_KEY,
   ].join(" ");
   console.log(`running command '${command}' in ${cwd}`);
   const appProcess = exec(command, {
     cwd,
     env: {
       ...process.env,
+      LLAMA_CLOUD_PROJECT_NAME: llamaCloudProjectName,
+      LLAMA_CLOUD_INDEX_NAME: llamaCloudIndexName,
     },
   });
   appProcess.stderr?.on("data", (data) => {

helpers/index.ts

@@ -23,6 +23,31 @@ import {
 } from "./types";
 import { installTSTemplate } from "./typescript";
 
+const checkForGenerateScript = (
+  modelConfig: ModelConfig,
+  vectorDb?: TemplateVectorDB,
+  llamaCloudKey?: string,
+  useLlamaParse?: boolean,
+) => {
+  const missingSettings = [];
+
+  if (!modelConfig.isConfigured()) {
+    missingSettings.push("your model provider API key");
+  }
+
+  const llamaCloudApiKey = llamaCloudKey ?? process.env["LLAMA_CLOUD_API_KEY"];
+  const isRequiredLlamaCloudKey = useLlamaParse || vectorDb === "llamacloud";
+  if (isRequiredLlamaCloudKey && !llamaCloudApiKey) {
+    missingSettings.push("your LLAMA_CLOUD_API_KEY");
+  }
+
+  if (vectorDb !== "none" && vectorDb !== "llamacloud") {
+    missingSettings.push("your Vector DB environment variables");
+  }
+
+  return missingSettings;
+};
+
 // eslint-disable-next-line max-params
 async function generateContextData(
   framework: TemplateFramework,
@@ -38,12 +63,15 @@ async function generateContextData(
         ? "poetry run generate"
         : `${packageManager} run generate`,
     )}`;
-    const modelConfigured = modelConfig.isConfigured();
-    const llamaCloudKeyConfigured = useLlamaParse
-      ? llamaCloudKey || process.env["LLAMA_CLOUD_API_KEY"]
-      : true;
-    const hasVectorDb = vectorDb && vectorDb !== "none";
-    if (modelConfigured && llamaCloudKeyConfigured && !hasVectorDb) {
+
+    const missingSettings = checkForGenerateScript(
+      modelConfig,
+      vectorDb,
+      llamaCloudKey,
+      useLlamaParse,
+    );
+
+    if (!missingSettings.length) {
       // If all the required environment variables are set, run the generate script
       if (framework === "fastapi") {
         if (isHavingPoetryLockFile()) {
@@ -63,15 +91,8 @@ async function generateContextData(
       }
     }
 
-    // generate the message of what to do to run the generate script manually
-    const settings = [];
-    if (!modelConfigured) settings.push("your model provider API key");
-    if (!llamaCloudKeyConfigured) settings.push("your Llama Cloud key");
-    if (hasVectorDb) settings.push("your Vector DB environment variables");
-    const settingsMessage =
-      settings.length > 0 ? `After setting ${settings.join(" and ")}, ` : "";
-    const generateMessage = `run ${runGenerate} to generate the context data.`;
-    console.log(`\n${settingsMessage}${generateMessage}\n\n`);
+    const settingsMessage = `After setting ${missingSettings.join(" and ")}, run ${runGenerate} to generate the context data.`;
+    console.log(`\n${settingsMessage}\n\n`);
   }
 }
 

index.ts

@@ -9,7 +9,7 @@ import prompts from "prompts";
 import terminalLink from "terminal-link";
 import checkForUpdate from "update-check";
 import { createApp } from "./create-app";
-import { getDataSources } from "./helpers/datasources";
+import { EXAMPLE_FILE, getDataSources } from "./helpers/datasources";
 import { getPkgManager } from "./helpers/get-pkg-manager";
 import { isFolderEmpty } from "./helpers/is-folder-empty";
 import { initializeGlobalAgent } from "./helpers/proxy";
@@ -194,8 +194,16 @@ if (process.argv.includes("--no-llama-parse")) {
 program.askModels = process.argv.includes("--ask-models");
 if (process.argv.includes("--no-files")) {
   program.dataSources = [];
-} else {
+} else if (process.argv.includes("--example-file")) {
   program.dataSources = getDataSources(program.files, program.exampleFile);
+} else if (process.argv.includes("--llamacloud")) {
+  program.dataSources = [
+    {
+      type: "llamacloud",
+      config: {},
+    },
+    EXAMPLE_FILE,
+  ];
 }
 
 const packageManager = !!program.useNpm

questions.ts

@@ -671,21 +671,24 @@ export const askQuestions = async (
 
   // Ask for LlamaCloud API key when using a LlamaCloud index or LlamaParse
   if (isUsingLlamaCloud || program.useLlamaParse) {
-    if (ciInfo.isCI) {
-      program.llamaCloudKey = getPrefOrDefault("llamaCloudKey");
-    } else {
-      // Ask for LlamaCloud API key
-      const { llamaCloudKey } = await prompts(
-        {
-          type: "text",
-          name: "llamaCloudKey",
-          message:
-            "Please provide your LlamaCloud API key (leave blank to skip):",
-        },
-        questionHandlers,
-      );
-      program.llamaCloudKey = preferences.llamaCloudKey =
-        llamaCloudKey || process.env.LLAMA_CLOUD_API_KEY;
+    if (!program.llamaCloudKey) {
+      // if already set, don't ask again
+      if (ciInfo.isCI) {
+        program.llamaCloudKey = getPrefOrDefault("llamaCloudKey");
+      } else {
+        // Ask for LlamaCloud API key
+        const { llamaCloudKey } = await prompts(
+          {
+            type: "text",
+            name: "llamaCloudKey",
+            message:
+              "Please provide your LlamaCloud API key (leave blank to skip):",
+          },
+          questionHandlers,
+        );
+        program.llamaCloudKey = preferences.llamaCloudKey =
+          llamaCloudKey || process.env.LLAMA_CLOUD_API_KEY;
+      }
     }
   }