feat(js): Allow overriding test suite name for Jest test suites, auto…

…matically set environment on project metadata (#1492)

js/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langsmith",
-  "version": "0.3.5",
+  "version": "0.3.6",
   "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.",
   "packageManager": "[email protected]",
   "files": [

js/src/index.ts

@@ -18,4 +18,4 @@ export { RunTree, type RunTreeConfig } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
 
 // Update using yarn bump-version
-export const __version__ = "0.3.5";
+export const __version__ = "0.3.6";

js/src/jest/index.ts

@@ -304,12 +304,6 @@ export {
    * ```
    */
   expect,
-  /** Whether the actual string value is close to the expected value in relative terms. */
-  toBeRelativeCloseTo,
-  /** Whether the actual string value is close to the expected value in absolute terms. */
-  toBeAbsoluteCloseTo,
-  /** Whether the actual string value is close to the expected value as scored by an embeddings model. */
-  toBeSemanticCloseTo,
   /**
    * Log feedback associated with the current test, usually generated by some kind of
    * evaluator.

js/src/tests/jestlike/vitest.vitesteval.ts

@@ -127,5 +127,6 @@ ls.describe(
     metadata: {
       model: "test-model",
     },
+    testSuiteName: "js-vitest-testing",
   }
 );

js/src/utils/jestlike/index.ts

@@ -32,8 +32,10 @@ import { SimpleEvaluationResult } from "./types.js";
 import type {
   LangSmithJestlikeWrapperConfig,
   LangSmithJestlikeWrapperParams,
-  LangSmithJestDescribeWrapper,
+  LangSmithJestlikeDescribeWrapper,
+  LangSmithJestlikeDescribeWrapperConfig,
 } from "./types.js";
+import { getEnvironmentVariable } from "../env.js";
 
 const DEFAULT_TEST_TIMEOUT = 30_000;
 
@@ -264,7 +266,7 @@ export function generateWrapperFromJestlikeMethods(
       });
       const experimentUrl = `${datasetUrl}/compare?selectedSessions=${project.id}`;
       console.log(
-        `[LANGSMITH]: Experiment starting! View results at ${experimentUrl}`
+        `[LANGSMITH]: Experiment starting for dataset "${datasetName}"!\n[LANGSMITH]: View results at ${experimentUrl}`
       );
       storageValue = {
         dataset,
@@ -278,30 +280,48 @@ export function generateWrapperFromJestlikeMethods(
 
   function wrapDescribeMethod(
     method: (name: string, fn: () => void | Promise<void>) => void
-  ): LangSmithJestDescribeWrapper {
+  ): LangSmithJestlikeDescribeWrapper {
     return function (
-      datasetName: string,
+      testSuiteName: string,
       fn: () => void | Promise<void>,
-      experimentConfig?: {
-        client?: Client;
-        enableTestTracking?: boolean;
-      } & Partial<Omit<CreateProjectParams, "referenceDatasetId">>
+      experimentConfig?: LangSmithJestlikeDescribeWrapperConfig
     ) {
       const client = experimentConfig?.client ?? DEFAULT_TEST_CLIENT;
-      return method(datasetName, () => {
+      const suiteName = experimentConfig?.testSuiteName ?? testSuiteName;
+      return method(suiteName, () => {
         const startTime = new Date();
         const suiteUuid = v4();
+        const environment =
+          experimentConfig?.metadata?.ENVIRONMENT ??
+          getEnvironmentVariable("ENVIRONMENT");
+        const nodeEnv =
+          experimentConfig?.metadata?.NODE_ENV ??
+          getEnvironmentVariable("NODE_ENV");
+        const langsmithEnvironment =
+          experimentConfig?.metadata?.LANGSMITH_ENVIRONMENT ??
+          getEnvironmentVariable("LANGSMITH_ENVIRONMENT");
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const metadata: Record<string, any> = {
+          ...experimentConfig?.metadata,
+          __ls_runner: testRunnerName,
+        };
+        if (environment !== undefined) {
+          metadata.ENVIRONMENT = environment;
+        }
+        if (nodeEnv !== undefined) {
+          metadata.NODE_ENV = nodeEnv;
+        }
+        if (langsmithEnvironment !== undefined) {
+          metadata.LANGSMITH_ENVIRONMENT = langsmithEnvironment;
+        }
         const context = {
           suiteUuid,
-          suiteName: datasetName,
+          suiteName,
           client,
           createdAt: new Date().toISOString(),
           projectConfig: {
             ...experimentConfig,
-            metadata: {
-              ...experimentConfig?.metadata,
-              __ls_runner: testRunnerName,
-            },
+            metadata,
           },
           enableTestTracking: experimentConfig?.enableTestTracking,
         };

js/src/utils/jestlike/types.ts

@@ -1,4 +1,6 @@
+import { CreateProjectParams } from "../../client.js";
 import { EvaluationResult } from "../../evaluation/evaluator.js";
+import { Client } from "../../index.js";
 import type { RunTreeConfig } from "../../run_trees.js";
 import type { SimpleEvaluator } from "./vendor/evaluatedBy.js";
 
@@ -17,12 +19,21 @@ export type LangSmithJestlikeWrapperParams<I, O> = {
   config?: LangSmithJestlikeWrapperConfig;
 };
 
-export type LangSmithJestDescribeWrapper = (
+export type LangSmithJestlikeDescribeWrapperConfig = {
+  client?: Client;
+  enableTestTracking?: boolean;
+  testSuiteName?: string;
+} & Partial<Omit<CreateProjectParams, "referenceDatasetId">>;
+
+export type LangSmithJestlikeDescribeWrapper = (
   name: string,
   fn: () => void | Promise<void>,
-  config?: Partial<RunTreeConfig>
+  config?: LangSmithJestlikeDescribeWrapperConfig
 ) => void;
 
+/** @deprecated Import as `LangSmithJestlikeDescribeWrapper` instead. */
+export type LangSmithJestDescribeWrapper = LangSmithJestlikeDescribeWrapper;
+
 export type SimpleEvaluationResult = {
   key: EvaluationResult["key"];
   score: NonNullable<EvaluationResult["score"]>;

js/src/vitest/index.ts

@@ -292,12 +292,6 @@ export {
    * ```
    */
   expect,
-  /** Whether the actual string value is close to the expected value in relative terms. */
-  toBeRelativeCloseTo,
-  /** Whether the actual string value is close to the expected value in absolute terms. */
-  toBeAbsoluteCloseTo,
-  /** Whether the actual string value is close to the expected value as scored by an embeddings model. */
-  toBeSemanticCloseTo,
   /**
    * Log feedback associated with the current test, usually generated by some kind of
    * evaluator.