use pipeline transformation & upgrade llamaindex latest

Author: thucpn

templates/types/streaming/express/package.json

@@ -20,7 +20,7 @@
     "dotenv": "^16.3.1",
     "duck-duck-scrape": "^2.2.5",
     "express": "^4.18.2",
-    "llamaindex": "0.4.3",
+    "llamaindex": "0.4.4",
     "pdf2json": "3.0.5",
     "ajv": "^8.12.0",
     "@e2b/code-interpreter": "^0.0.5",

templates/types/streaming/nextjs/app/api/chat/embed/embeddings.ts

@@ -1,32 +1,27 @@
-import { Document, MetadataMode, Settings, SimpleNodeParser } from "llamaindex";
+import {
+  Document,
+  IngestionPipeline,
+  MetadataMode,
+  Settings,
+  SimpleNodeParser,
+} from "llamaindex";
 import pdf from "pdf-parse";
 
 export async function splitAndEmbed(content: string) {
-  // const document = new Document({ text: content, id_: "123" });
-  // const pipeline = new IngestionPipeline({
-  //   transformations: [
-  //     new SimpleNodeParser({
-  //       chunkSize: Settings.chunkSize,
-  //       chunkOverlap: Settings.chunkOverlap,
-  //     }),
-  //     Settings.embedModel,
-  //   ],
-  // });
-  // const nodes = await pipeline.run({ documents: [document] });
-  // return nodes;
-
-  const nodeParser = new SimpleNodeParser({
-    chunkSize: Settings.chunkSize,
-    chunkOverlap: Settings.chunkOverlap,
+  const document = new Document({ text: content });
+  const pipeline = new IngestionPipeline({
+    transformations: [
+      new SimpleNodeParser({
+        chunkSize: Settings.chunkSize,
+        chunkOverlap: Settings.chunkOverlap,
+      }),
+      Settings.embedModel,
+    ],
   });
-  const nodes = nodeParser.getNodesFromDocuments([
-    new Document({ text: content }),
-  ]);
-  const texts = nodes.map((node) => node.getContent(MetadataMode.EMBED));
-  const embeddings = await Settings.embedModel.getTextEmbeddingsBatch(texts);
+  const nodes = await pipeline.run({ documents: [document] });
   return nodes.map((node, i) => ({
     text: node.getContent(MetadataMode.NONE),
-    embedding: embeddings[i],
+    embedding: node.embedding,
   }));
 }
 

templates/types/streaming/nextjs/package.json

@@ -24,7 +24,7 @@
     "duck-duck-scrape": "^2.2.5",
     "formdata-node": "^6.0.3",
     "got": "^14.4.1",
-    "llamaindex": "0.4.3",
+    "llamaindex": "0.4.4",
     "lucide-react": "^0.294.0",
     "next": "^14.2.4",
     "pdf2json": "3.0.5",
@@ -60,6 +60,7 @@
     "prettier-plugin-organize-imports": "^3.2.4",
     "tailwindcss": "^3.3.6",
     "tsx": "^4.7.2",
-    "typescript": "^5.3.2"
+    "typescript": "^5.3.2",
+    "@types/pdf-parse": "^1.1.4"
   }
 }