feat: init @llamaindex/core (#938)

Author: himself65

.github/workflows/test.yml

@@ -128,6 +128,9 @@ jobs:
         run: pnpm run build
       - name: Copy examples
         run: rsync -rv --exclude=node_modules ./examples ${{ runner.temp }}
+      - name: Pack @llamaindex/core
+        run: pnpm pack --pack-destination ${{ runner.temp }}
+        working-directory: packages/core
       - name: Pack @llamaindex/env
         run: pnpm pack --pack-destination ${{ runner.temp }}
         working-directory: packages/env

package.json

@@ -24,14 +24,14 @@
     "eslint": "^8.57.0",
     "eslint-config-next": "^14.2.4",
     "eslint-config-prettier": "^9.1.0",
-    "eslint-config-turbo": "^1.13.4",
+    "eslint-config-turbo": "^2.0.5",
     "eslint-plugin-react": "7.34.1",
     "husky": "^9.0.11",
     "lint-staged": "^15.2.7",
     "madge": "^7.0.0",
     "prettier": "^3.3.2",
     "prettier-plugin-organize-imports": "^3.2.4",
-    "turbo": "^1.13.4",
+    "turbo": "^2.0.5",
     "typescript": "^5.5.2"
   },
   "packageManager": "[email protected]",
@@ -40,6 +40,9 @@
       "trim": "1.0.1",
       "@babel/traverse": "7.23.2",
       "protobufjs": "7.2.6"
+    },
+    "patchedDependencies": {
+      "[email protected]": "patches/[email protected]"
     }
   },
   "lint-staged": {

packages/community/jsr.json

@@ -0,0 +1,67 @@
+{
+  "name": "@llamaindex/core",
+  "type": "module",
+  "version": "0.0.1",
+  "description": "LlamaIndex Core Module",
+  "exports": {
+    "./decorator": {
+      "require": {
+        "types": "./dist/decorator/index.d.cts",
+        "default": "./dist/decorator/index.cjs"
+      },
+      "import": {
+        "types": "./dist/decorator/index.d.ts",
+        "default": "./dist/decorator/index.js"
+      },
+      "default": {
+        "types": "./dist/decorator/index.d.ts",
+        "default": "./dist/decorator/index.js"
+      }
+    },
+    "./global": {
+      "require": {
+        "types": "./dist/global/index.d.cts",
+        "default": "./dist/global/index.cjs"
+      },
+      "import": {
+        "types": "./dist/global/index.d.ts",
+        "default": "./dist/global/index.js"
+      },
+      "default": {
+        "types": "./dist/global/index.d.ts",
+        "default": "./dist/global/index.js"
+      }
+    },
+    "./schema": {
+      "require": {
+        "types": "./dist/schema/index.d.cts",
+        "default": "./dist/schema/index.cjs"
+      },
+      "import": {
+        "types": "./dist/schema/index.d.ts",
+        "default": "./dist/schema/index.js"
+      },
+      "default": {
+        "types": "./dist/schema/index.d.ts",
+        "default": "./dist/schema/index.js"
+      }
+    }
+  },
+  "scripts": {
+    "dev": "bunchee --watch",
+    "build": "bunchee"
+  },
+  "repository": {
+    "type": "git",
+    "directory": "packages/core",
+    "url": "https://github.com/himself65/LlamaIndexTS.git"
+  },
+  "devDependencies": {
+    "bunchee": "^5.2.1"
+  },
+  "dependencies": {
+    "@llamaindex/env": "workspace:*",
+    "@types/node": "^20.14.9",
+    "zod": "^3.23.8"
+  }
+}

packages/core/package.json

@@ -1,11 +1,11 @@
 import { getEnv } from "@llamaindex/env";
-import type { BaseNode } from "../../Node.js";
-import { getChunkSize } from "../settings/chunk-size.js";
+import { Settings } from "../global";
+import type { BaseNode } from "../schema/node";
 
 const emitOnce = false;
 
 export function chunkSizeCheck<
-  This extends BaseNode,
+  This extends { id_: string },
   Args extends any[],
   Return,
 >(
@@ -17,15 +17,15 @@ export function chunkSizeCheck<
 ) {
   return function (this: This, ...args: Args) {
     const content = contentGetter.call(this, ...args);
-    const chunkSize = getChunkSize();
+    const chunkSize = Settings.chunkSize;
     const enableChunkSizeCheck = getEnv("ENABLE_CHUNK_SIZE_CHECK") === "true";
     if (
       enableChunkSizeCheck &&
       chunkSize !== undefined &&
       content.length > chunkSize
     ) {
       console.warn(
-        `Node (${this.id_}) is larger than chunk size: ${content.length}`,
+        `Node (${this.id_}) is larger than chunk size: ${content.length} > ${chunkSize}`,
       );
       if (!emitOnce) {
         console.warn(

packages/llamaindex/src/internal/decorator/node.ts renamed to packages/core/src/decorator/index.ts

@@ -0,0 +1 @@
+export { Settings } from "./settings";

packages/core/src/global/index.ts

@@ -0,0 +1,17 @@
+import {
+  getChunkSize,
+  setChunkSize,
+  withChunkSize,
+} from "./settings/chunk-size";
+
+export const Settings = {
+  get chunkSize(): number | undefined {
+    return getChunkSize();
+  },
+  set chunkSize(chunkSize: number | undefined) {
+    setChunkSize(chunkSize);
+  },
+  withChunkSize<Result>(chunkSize: number, fn: () => Result): Result {
+    return withChunkSize(chunkSize, fn);
+  },
+};

packages/core/src/global/settings.ts

@@ -1,14 +1,16 @@
 import { AsyncLocalStorage } from "@llamaindex/env";
 
 const chunkSizeAsyncLocalStorage = new AsyncLocalStorage<number | undefined>();
-const globalChunkSize: number | null = null;
+let globalChunkSize: number | null = null;
 
 export function getChunkSize(): number | undefined {
   return globalChunkSize ?? chunkSizeAsyncLocalStorage.getStore();
 }
 
 export function setChunkSize(chunkSize: number | undefined) {
-  chunkSizeAsyncLocalStorage.enterWith(chunkSize);
+  if (chunkSize !== undefined) {
+    globalChunkSize = chunkSize;
+  }
 }
 
 export function withChunkSize<Result>(

packages/llamaindex/src/internal/settings/chunk-size.ts renamed to packages/core/src/global/settings/chunk-size.ts

@@ -0,0 +1 @@
+export * from "./schema";

packages/core/src/index.ts

@@ -0,0 +1,2 @@
+export * from "./node";
+export * from "./zod";