feat(js): Vitest/Jest evals (#1343)

langchain-ai · Jan 20, 2025 · 98c9009 · 98c9009
1 parent 04d87c8
commit 98c9009
Show file tree

Hide file tree

Showing 28 changed files with 4,018 additions and 19 deletions.
diff --git a/.github/actions/js-vitest-eval-test/action.yml b/.github/actions/js-vitest-eval-test/action.yml
@@ -0,0 +1,35 @@
+name: "JS Vitest Eval Runner"
+description: "Run JS Vitest Eval Runner"
+inputs:
+  node-version:
+    description: "Node version"
+    required: true
+  langchain-api-key-beta:
+    description: "Langchain"
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+
+    - name: Setup Node.js
+      uses: actions/setup-node@v3
+      with:
+        node-version: ${{ inputs.node-version }}
+        cache: "yarn"
+        cache-dependency-path: "js/yarn.lock"
+
+    - name: Install Yarn dependencies
+      run: yarn install
+      shell: bash
+      working-directory: js
+
+    - name: Run Vitest Eval Runner
+      run: yarn test:eval:vitest
+      shell: bash
+      working-directory: js
+      env:
+        LANGSMITH_TRACING_V2: "true"
+        LANGSMITH_ENDPOINT: https://beta.api.smith.langchain.com
+        LANGSMITH_API_KEY: ${{ inputs.langchain-api-key-beta }}
diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
@@ -103,3 +103,34 @@ jobs:
           node-version: 20.x
           langchain-api-key-beta: ${{ secrets.LANGSMITH_API_KEY_BETA }}
           openai-api-key: ${{ secrets.OPENAI_API_KEY }}
+
+  js_vitest_eval_runner_test:
+    name: JS Vitest Runner Test
+    needs: changed_files
+    if: >
+      (github.event_name == 'push') ||
+      (github.event_name == 'pull_request' && (
+        contains(github.event.pull_request.labels.*.name, 'release') ||
+        needs.changed_files.outputs.js_changed == 'true'
+      )) ||
+      (github.event_name == 'workflow_dispatch' && github.event.inputs.run-js-tests == 'true')
+    runs-on: ubuntu-20.04
+    defaults:
+      run:
+        working-directory: js
+    steps:
+      - uses: actions/checkout@v3
+      - name: Use Node.js 18.x
+        uses: actions/setup-node@v3
+        with:
+          node-version: 20.x
+          cache: "yarn"
+          cache-dependency-path: "js/yarn.lock"
+      - name: Install dependencies
+        run: yarn install --immutable
+      - name: Run JS Vitest eval runner test
+        uses: ./.github/actions/js-vitest-eval-test
+        with:
+          node-version: 20.x
+          langchain-api-key-beta: ${{ secrets.LANGSMITH_API_KEY_BETA }}
+          openai-api-key: ${{ secrets.OPENAI_API_KEY }}
diff --git a/js/.gitignore b/js/.gitignore
@@ -59,10 +59,26 @@ Chinook_Sqlite.sql
 /langchain.js
 /langchain.d.ts
 /langchain.d.cts
+/jest.cjs
+/jest.js
+/jest.d.ts
+/jest.d.cts
+/jest/reporter.cjs
+/jest/reporter.js
+/jest/reporter.d.ts
+/jest/reporter.d.cts
 /vercel.cjs
 /vercel.js
 /vercel.d.ts
 /vercel.d.cts
+/vitest.cjs
+/vitest.js
+/vitest.d.ts
+/vitest.d.cts
+/vitest/reporter.cjs
+/vitest/reporter.js
+/vitest/reporter.d.ts
+/vitest/reporter.d.cts
 /wrappers.cjs
 /wrappers.js
 /wrappers.d.ts

diff --git a/js/ls.vitest.config.ts b/js/ls.vitest.config.ts
@@ -0,0 +1,9 @@
+import { defineConfig } from "vitest/config";
+
+export default defineConfig({
+  test: {
+    include: ["**/*.vitesteval.?(c|m)[jt]s"],
+    reporters: ["./src/vitest/reporter.ts"],
+    setupFiles: ["dotenv/config"],
+  },
+});
diff --git a/js/package.json b/js/package.json
@@ -1,6 +1,6 @@
 {
   "name": "langsmith",
-  "version": "0.2.15",
+  "version": "0.3.0",
   "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.",
   "packageManager": "[email protected]",
   "files": [
@@ -33,10 +33,26 @@
     "langchain.js",
     "langchain.d.ts",
     "langchain.d.cts",
+    "jest.cjs",
+    "jest.js",
+    "jest.d.ts",
+    "jest.d.cts",
+    "jest/reporter.cjs",
+    "jest/reporter.js",
+    "jest/reporter.d.ts",
+    "jest/reporter.d.cts",
     "vercel.cjs",
     "vercel.js",
     "vercel.d.ts",
     "vercel.d.cts",
+    "vitest.cjs",
+    "vitest.js",
+    "vitest.d.ts",
+    "vitest.d.cts",
+    "vitest/reporter.cjs",
+    "vitest/reporter.js",
+    "vitest/reporter.d.ts",
+    "vitest/reporter.d.cts",
     "wrappers.cjs",
     "wrappers.js",
     "wrappers.d.ts",
@@ -78,6 +94,7 @@
     "test:integration": "cross-env NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.int\\.test.ts --testTimeout 100000",
     "test:single": "NODE_OPTIONS=--experimental-vm-modules yarn run jest --config jest.config.cjs --testTimeout 100000",
     "watch:single": "NODE_OPTIONS=--experimental-vm-modules yarn run jest --watch --config jest.config.cjs --testTimeout 100000",
+    "test:eval:vitest": "vitest run --config ls.vitest.config.ts",
     "lint": "NODE_OPTIONS=--max-old-space-size=4096 eslint --cache --ext .ts,.js src/",
     "lint:fix": "yarn lint --fix",
     "format": "prettier --write 'src/**/*.{ts,tsx}'",
@@ -103,7 +120,8 @@
   "homepage": "https://github.com/langchain-ai/langsmith-sdk#readme",
   "dependencies": {
     "@types/uuid": "^10.0.0",
-    "commander": "^10.0.1",
+    "chalk": "^4.1.2",
+    "console-table-printer": "^2.12.1",
     "p-queue": "^6.6.2",
     "p-retry": "4",
     "semver": "^7.6.3",
@@ -114,6 +132,7 @@
     "@babel/preset-env": "^7.22.4",
     "@faker-js/faker": "^8.4.1",
     "@jest/globals": "^29.5.0",
+    "@jest/reporters": "^29.7.0",
     "@langchain/core": "^0.3.14",
     "@langchain/langgraph": "^0.2.20",
     "@langchain/openai": "^0.3.11",
@@ -143,6 +162,7 @@
     "typedoc": "^0.27.6",
     "typedoc-plugin-expand-object-like-types": "^0.1.2",
     "typescript": "^5.4.5",
+    "vitest": "^2.1.8",
     "zod": "^3.23.8"
   },
   "peerDependencies": {
@@ -232,6 +252,24 @@
       "import": "./langchain.js",
       "require": "./langchain.cjs"
     },
+    "./jest": {
+      "types": {
+        "import": "./jest.d.ts",
+        "require": "./jest.d.cts",
+        "default": "./jest.d.ts"
+      },
+      "import": "./jest.js",
+      "require": "./jest.cjs"
+    },
+    "./jest/reporter": {
+      "types": {
+        "import": "./jest/reporter.d.ts",
+        "require": "./jest/reporter.d.cts",
+        "default": "./jest/reporter.d.ts"
+      },
+      "import": "./jest/reporter.js",
+      "require": "./jest/reporter.cjs"
+    },
     "./vercel": {
       "types": {
         "import": "./vercel.d.ts",
@@ -241,6 +279,24 @@
       "import": "./vercel.js",
       "require": "./vercel.cjs"
     },
+    "./vitest": {
+      "types": {
+        "import": "./vitest.d.ts",
+        "require": "./vitest.d.cts",
+        "default": "./vitest.d.ts"
+      },
+      "import": "./vitest.js",
+      "require": "./vitest.cjs"
+    },
+    "./vitest/reporter": {
+      "types": {
+        "import": "./vitest/reporter.d.ts",
+        "require": "./vitest/reporter.d.cts",
+        "default": "./vitest/reporter.d.ts"
+      },
+      "import": "./vitest/reporter.js",
+      "require": "./vitest/reporter.cjs"
+    },
     "./wrappers": {
       "types": {
         "import": "./wrappers.d.ts",

diff --git a/js/scripts/create-entrypoints.js b/js/scripts/create-entrypoints.js
@@ -14,14 +14,22 @@ const entrypoints = {
   "evaluation/langchain": "evaluation/langchain",
   schemas: "schemas",
   langchain: "langchain",
+  jest: "jest/index",
+  "jest/reporter": "jest/reporter",
   vercel: "vercel",
+  vitest: "vitest/index",
+  "vitest/reporter": "vitest/reporter",
   wrappers: "wrappers/index",
   anonymizer: "anonymizer/index",
   "wrappers/openai": "wrappers/openai",
   "wrappers/vercel": "wrappers/vercel",
   "singletons/traceable": "singletons/traceable",
 };
 
+const defaultEntrypoints = [
+  "vitest/reporter"
+];
+
 const updateJsonFile = (relativePath, updateFunction) => {
   const contents = fs.readFileSync(relativePath).toString();
   const res = updateFunction(JSON.parse(contents));
@@ -34,6 +42,17 @@ const generateFiles = () => {
       const nrOfDots = key.split("/").length - 1;
       const relativePath = "../".repeat(nrOfDots) || "./";
       const compiledPath = `${relativePath}dist/${value}.js`;
+      if (defaultEntrypoints.includes(key)) {
+        return [
+          [
+            `${key}.cjs`,
+            `module.exports = require('${relativePath}dist/${value}.cjs').default;`,
+          ],
+          [`${key}.js`, `export { default } from '${compiledPath}'`],
+          [`${key}.d.ts`, `export { default } from '${compiledPath}'`],
+          [`${key}.d.cts`, `export { default } from '${compiledPath}'`],
+        ];
+      }
       return [
         [
           `${key}.cjs`,