Skip to content

Commit 2a43ebd

Browse files
marcusschiessermarcusschiesser
committed
split e2e tests in python and ts
1 parent fe489db commit 2a43ebd

8 files changed

+192
-134
lines changed

‎.github/workflows/e2e.yml

Lines changed: 73 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,16 +9,83 @@ env:
99
POETRY_VERSION: "1.6.1"
1010

1111
jobs:
12-
e2e:
13-
name: create-llama
12+
e2e-python:
13+
name: create-llama-python
14+
timeout-minutes: 60
15+
strategy:
16+
fail-fast: true
17+
matrix:
18+
node-version: [20]
19+
python-version: ["3.11"]
20+
os: [macos-latest, windows-latest, ubuntu-22.04]
21+
frameworks: ["fastapi"]
22+
datasources: ["--no-files", "--example-file"]
23+
defaults:
24+
run:
25+
shell: bash
26+
runs-on: ${{ matrix.os }}
27+
steps:
28+
- uses: actions/checkout@v4
29+
30+
- name: Set up python ${{ matrix.python-version }}
31+
uses: actions/setup-python@v5
32+
with:
33+
python-version: ${{ matrix.python-version }}
34+
35+
- name: Install Poetry
36+
uses: snok/install-poetry@v1
37+
with:
38+
version: ${{ env.POETRY_VERSION }}
39+
40+
- uses: pnpm/action-setup@v3
41+
42+
- name: Setup Node.js ${{ matrix.node-version }}
43+
uses: actions/setup-node@v4
44+
with:
45+
node-version: ${{ matrix.node-version }}
46+
cache: "pnpm"
47+
48+
- name: Install dependencies
49+
run: pnpm install
50+
51+
- name: Install Playwright Browsers
52+
run: pnpm exec playwright install --with-deps
53+
working-directory: .
54+
55+
- name: Build create-llama
56+
run: pnpm run build
57+
working-directory: .
58+
59+
- name: Install
60+
run: pnpm run pack-install
61+
working-directory: .
62+
63+
- name: Run Playwright tests for Python
64+
run: pnpm run e2e:python
65+
env:
66+
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
67+
LLAMA_CLOUD_API_KEY: ${{ secrets.LLAMA_CLOUD_API_KEY }}
68+
FRAMEWORK: ${{ matrix.frameworks }}
69+
DATASOURCE: ${{ matrix.datasources }}
70+
working-directory: .
71+
72+
- uses: actions/upload-artifact@v3
73+
if: always()
74+
with:
75+
name: playwright-report-python
76+
path: ./playwright-report/
77+
retention-days: 30
78+
79+
e2e-typescript:
80+
name: create-llama-typescript
1481
timeout-minutes: 60
1582
strategy:
1683
fail-fast: true
1784
matrix:
1885
node-version: [18, 20]
1986
python-version: ["3.11"]
2087
os: [macos-latest, windows-latest, ubuntu-22.04]
21-
frameworks: ["nextjs", "express", "fastapi"]
88+
frameworks: ["nextjs", "express"]
2289
datasources: ["--no-files", "--example-file"]
2390
defaults:
2491
run:
@@ -60,8 +127,8 @@ jobs:
60127
run: pnpm run pack-install
61128
working-directory: .
62129

63-
- name: Run Playwright tests
64-
run: pnpm run e2e
130+
- name: Run Playwright tests for TypeScript
131+
run: pnpm run e2e:typescript
65132
env:
66133
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
67134
LLAMA_CLOUD_API_KEY: ${{ secrets.LLAMA_CLOUD_API_KEY }}
@@ -72,6 +139,6 @@ jobs:
72139
- uses: actions/upload-artifact@v3
73140
if: always()
74141
with:
75-
name: playwright-report
142+
name: playwright-report-typescript
76143
path: ./playwright-report/
77144
retention-days: 30

‎e2e/resolve_python_dependencies.spec.ts renamed to ‎e2e/python/resolve_dependencies.spec.ts

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,21 +3,16 @@ import { exec } from "child_process";
33
import fs from "fs";
44
import path from "path";
55
import util from "util";
6-
import { TemplateFramework, TemplateVectorDB } from "../helpers/types";
7-
import { createTestDir, runCreateLlama } from "./utils";
6+
import { TemplateVectorDB } from "../../helpers/types";
7+
import { createTestDir, runCreateLlama } from "../utils";
88

99
const execAsync = util.promisify(exec);
1010

11-
const templateFramework: TemplateFramework = process.env.FRAMEWORK
12-
? (process.env.FRAMEWORK as TemplateFramework)
13-
: "fastapi";
1411
const dataSource: string = process.env.DATASOURCE
1512
? process.env.DATASOURCE
1613
: "--example-file";
1714

1815
if (
19-
templateFramework == "fastapi" && // test is only relevant for fastapi
20-
process.version.startsWith("v20.") && // XXX: Only run for Node.js version 20 (CI matrix will trigger other versions)
2116
dataSource === "--example-file" // XXX: this test provides its own data source - only trigger it on one data source (usually the CI matrix will trigger multiple data sources)
2217
) {
2318
// vectorDBs, tools, and data source combinations to test

‎e2e/resolve_ts_dependencies.spec.ts

Lines changed: 0 additions & 112 deletions
This file was deleted.

‎e2e/extractor_template.spec.ts renamed to ‎e2e/shared/extractor_template.spec.ts

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@ import { expect, test } from "@playwright/test";
33
import { ChildProcess } from "child_process";
44
import fs from "fs";
55
import path from "path";
6-
import { TemplateFramework } from "../helpers";
7-
import { createTestDir, runCreateLlama } from "./utils";
6+
import { TemplateFramework } from "../../helpers";
7+
import { createTestDir, runCreateLlama } from "../utils";
88

99
const templateFramework: TemplateFramework = process.env.FRAMEWORK
1010
? (process.env.FRAMEWORK as TemplateFramework)
@@ -16,9 +16,8 @@ const dataSource: string = process.env.DATASOURCE
1616
// The extractor template currently only works with FastAPI and files (and not on Windows)
1717
if (
1818
process.platform !== "win32" &&
19-
templateFramework !== "nextjs" &&
20-
templateFramework !== "express" &&
21-
dataSource !== "--no-files"
19+
templateFramework === "fastapi" &&
20+
dataSource === "--example-file"
2221
) {
2322
test.describe("Test extractor template", async () => {
2423
let frontendPort: number;

‎e2e/multiagent_template.spec.ts renamed to ‎e2e/shared/multiagent_template.spec.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@ import type {
77
TemplateFramework,
88
TemplatePostInstallAction,
99
TemplateUI,
10-
} from "../helpers";
11-
import { createTestDir, runCreateLlama, type AppType } from "./utils";
10+
} from "../../helpers";
11+
import { createTestDir, runCreateLlama, type AppType } from "../utils";
1212

1313
const templateFramework: TemplateFramework = process.env.FRAMEWORK
1414
? (process.env.FRAMEWORK as TemplateFramework)

‎e2e/streaming_template.spec.ts renamed to ‎e2e/shared/streaming_template.spec.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@ import type {
77
TemplateFramework,
88
TemplatePostInstallAction,
99
TemplateUI,
10-
} from "../helpers";
11-
import { createTestDir, runCreateLlama, type AppType } from "./utils";
10+
} from "../../helpers";
11+
import { createTestDir, runCreateLlama, type AppType } from "../utils";
1212

1313
const templateFramework: TemplateFramework = process.env.FRAMEWORK
1414
? (process.env.FRAMEWORK as TemplateFramework)

‎e2e/typescript/resolve_dependencies.spec.ts

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
import { expect, test } from "@playwright/test";
2+
import { exec } from "child_process";
3+
import fs from "fs";
4+
import path from "path";
5+
import util from "util";
6+
import { TemplateFramework, TemplateVectorDB } from "../../helpers/types";
7+
import { createTestDir, runCreateLlama } from "../utils";
8+
9+
const execAsync = util.promisify(exec);
10+
11+
const templateFramework: TemplateFramework = process.env.FRAMEWORK
12+
? (process.env.FRAMEWORK as TemplateFramework)
13+
: "nextjs";
14+
const dataSource: string = process.env.DATASOURCE
15+
? process.env.DATASOURCE
16+
: "--example-file";
17+
18+
// vectorDBs combinations to test
19+
const vectorDbs: TemplateVectorDB[] = [
20+
"mongo",
21+
"pg",
22+
"qdrant",
23+
"pinecone",
24+
"milvus",
25+
"astra",
26+
"chroma",
27+
"llamacloud",
28+
"weaviate",
29+
];
30+
31+
test.describe("Test resolve TS dependencies", () => {
32+
// Test vector DBs without LlamaParse
33+
for (const vectorDb of vectorDbs) {
34+
const optionDescription = `vectorDb: ${vectorDb}, dataSource: ${dataSource}`;
35+
36+
test(`Vector DB test - ${optionDescription}`, async () => {
37+
await runTest(vectorDb, false);
38+
});
39+
}
40+
41+
// Test LlamaParse with vectorDB 'none'
42+
test(`LlamaParse test - vectorDb: none, dataSource: ${dataSource}, llamaParse: true`, async () => {
43+
await runTest("none", true);
44+
});
45+
46+
async function runTest(
47+
vectorDb: TemplateVectorDB | "none",
48+
useLlamaParse: boolean,
49+
) {
50+
const cwd = await createTestDir();
51+
52+
const result = await runCreateLlama({
53+
cwd: cwd,
54+
templateType: "streaming",
55+
templateFramework: templateFramework,
56+
dataSource: dataSource,
57+
vectorDb: vectorDb,
58+
port: 3000,
59+
externalPort: 8000,
60+
postInstallAction: "none",
61+
templateUI: undefined,
62+
appType: templateFramework === "nextjs" ? "" : "--no-frontend",
63+
llamaCloudProjectName: undefined,
64+
llamaCloudIndexName: undefined,
65+
tools: undefined,
66+
useLlamaParse: useLlamaParse,
67+
});
68+
const name = result.projectName;
69+
70+
// Check if the app folder exists
71+
const appDir = path.join(cwd, name);
72+
const dirExists = fs.existsSync(appDir);
73+
expect(dirExists).toBeTruthy();
74+
75+
// Install dependencies using pnpm
76+
try {
77+
const { stderr: installStderr } = await execAsync(
78+
"pnpm install --prefer-offline",
79+
{
80+
cwd: appDir,
81+
},
82+
);
83+
expect(installStderr).toBeFalsy();
84+
} catch (error) {
85+
console.error("Error installing dependencies:", error);
86+
throw error;
87+
}
88+
89+
// Run tsc type check and capture the output
90+
try {
91+
const { stdout, stderr } = await execAsync(
92+
"pnpm exec tsc -b --diagnostics",
93+
{
94+
cwd: appDir,
95+
},
96+
);
97+
// Check if there's any error output
98+
expect(stderr).toBeFalsy();
99+
100+
// Log the stdout for debugging purposes
101+
console.log("TypeScript type-check output:", stdout);
102+
} catch (error) {
103+
console.error("Error running tsc:", error);
104+
throw error;
105+
}
106+
}
107+
});

0 commit comments

Comments
 (0)