feat: Add e2e evals

.vscode/settings.json

@@ -1,3 +1,3 @@
 {
-  "cSpell.words": ["reranked", "Supabase"]
+  "cSpell.words": ["checkpointer", "reranked", "Supabase"]
 }

jest.setup.cjs

@@ -1,14 +1,14 @@
-const timezoneMock = require("timezone-mock");
+// const timezoneMock = require("timezone-mock");
 require("dotenv").config();
 
 // Mock the timezone to 'America/Los_Angeles'
-timezoneMock.register("US/Pacific"); // Alternatively, use 'America/Los_Angeles'
+// timezoneMock.register("US/Pacific"); // Alternatively, use 'America/Los_Angeles'
 
 // Optional: Log the current timezone to verify
-console.log(
-  "Current Timezone:",
-  Intl.DateTimeFormat().resolvedOptions().timeZone,
-);
+// console.log(
+//   "Current Timezone:",
+//   Intl.DateTimeFormat().resolvedOptions().timeZone,
+// );
 
 // If you have any global configurations or mocks, add them here
 // For example, you can set up global variables, mock APIs, etc.

package.json

@@ -54,12 +54,13 @@
     "express-session": "^1.18.1",
     "file-type": "^19.6.0",
     "google-auth-library": "^9.15.0",
-    "langsmith": "0.2.15-rc.2",
+    "langsmith": "0.2.15-rc.6",
     "moment": "^2.30.1",
     "passport": "^0.7.0",
     "passport-twitter": "^1.0.4",
     "playwright": "^1.49.1",
     "twitter-api-v2": "^1.18.2",
+    "uuid": "^11.0.4",
     "zod": "^3.23.8"
   },
   "devDependencies": {

src/agents/find-images/nodes/find-images.ts

@@ -8,15 +8,41 @@ import {
 } from "../../utils.js";
 import { takeScreenshotAndUpload } from "../screenshot.js";
 import { FindImagesAnnotation } from "../find-images-graph.js";
+import { validate } from "uuid";
+
+function checkIsGitHubImageUrl(url: string): boolean {
+  if (
+    url?.startsWith("https://github.com/user-attachments/assets") ||
+    url?.includes("githubusercontent.com/")
+  ) {
+    return true;
+  }
+  try {
+    const parsedUrl = new URL(url);
+    const pathname = parsedUrl.pathname;
+    const split = pathname.split("/");
+    const lastEle = split[split.length - 1];
+    const thirdToLastEle = split[split.length - 3];
+    if (thirdToLastEle === "assets" && validate(lastEle)) {
+      return true;
+    }
+    return false;
+  } catch (_) {
+    return false;
+  }
+}
 
 export async function findImages(state: typeof FindImagesAnnotation.State) {
   const { pageContents, imageOptions, relevantLinks } = state;
   const link = relevantLinks[0];
   const imageUrls = new Set<string>();
 
-  const screenshotUrl = await takeScreenshotAndUpload(link);
-  if (screenshotUrl) {
-    imageUrls.add(screenshotUrl);
+  let screenshotUrl: string | undefined;
+  if (getUrlType(link) !== "youtube") {
+    screenshotUrl = await takeScreenshotAndUpload(link);
+    if (screenshotUrl) {
+      imageUrls.add(screenshotUrl);
+    }
   }
 
   if (imageOptions.length) {
@@ -38,11 +64,7 @@ export async function findImages(state: typeof FindImagesAnnotation.State) {
           // If a full github URL. extract the file name from the path. to do this, extract the path after `blob/<branch>`
           const filePath = urlOrPathname.match(/blob\/[^/]+\/(.+)/)?.[1];
           if (!filePath) {
-            if (
-              !urlOrPathname?.startsWith(
-                "https://github.com/user-attachments/assets",
-              )
-            ) {
+            if (!checkIsGitHubImageUrl(urlOrPathname)) {
               console.warn(
                 "Could not extract file path from URL",
                 urlOrPathname,

src/agents/find-images/nodes/re-rank-images.ts

@@ -58,9 +58,10 @@ export function parseResult(result: string): number[] {
 }
 
 export async function reRankImages(state: typeof FindImagesAnnotation.State) {
-  if (state.imageOptions.length === 0) {
+  // No need to re-rank if less than 2 images
+  if (state.imageOptions.length < 2) {
     return {
-      imageOptions: [],
+      imageOptions: state.imageOptions,
     };
   }
 
@@ -86,21 +87,38 @@ export async function reRankImages(state: typeof FindImagesAnnotation.State) {
     if (!imageMessages.length) {
       continue;
     }
-    const response = await model.invoke([
-      {
-        role: "system",
-        content: formattedSystemPrompt,
-      },
-      {
-        role: "user",
-        content: imageMessages,
-      },
-    ]);
-
-    const chunkAnalysis = parseResult(response.content as string);
-    // Convert chunk indices to global indices and add to our list of re-ranked indices
-    const globalIndices = chunkAnalysis.map((index) => index + baseIndex);
-    reRankedIndices = [...reRankedIndices, ...globalIndices];
+
+    try {
+      const response = await model.invoke([
+        {
+          role: "system",
+          content: formattedSystemPrompt,
+        },
+        {
+          role: "user",
+          content: imageMessages,
+        },
+      ]);
+
+      const chunkAnalysis = parseResult(response.content as string);
+      // Convert chunk indices to global indices and add to our list of re-ranked indices
+      const globalIndices = chunkAnalysis.map((index) => index + baseIndex);
+      reRankedIndices = [...reRankedIndices, ...globalIndices];
+    } catch (error) {
+      console.error(
+        `Failed to re-rank images.\nImage URLs: ${imageMessages
+          .filter((m) => m.fileUri)
+          .map((m) => m.fileUri)
+          .join(", ")}\n\nError:`,
+        error,
+      );
+      // Add all indices from the failed chunk to allIrrelevantIndices
+      const failedChunkIndices = Array.from(
+        { length: imageChunk.length },
+        (_, i) => i + baseIndex,
+      );
+      reRankedIndices = [...reRankedIndices, ...failedChunkIndices];
+    }
 
     baseIndex += imageChunk.length;
   }

src/agents/find-images/nodes/validate-images.ts

@@ -56,6 +56,8 @@ export function parseResult(result: string): number[] {
     .filter((n) => !isNaN(n));
 }
 
+const YOUTUBE_THUMBNAIL_URL = "https://i.ytimg.com/";
+
 export async function validateImages(
   state: typeof FindImagesAnnotation.State,
 ): Promise<{
@@ -68,22 +70,21 @@ export async function validateImages(
     temperature: 0,
   });
 
-  let imagesWithoutSupabase = imageOptions;
-  if (process.env.SUPABASE_URL) {
-    // Do not include the supabase screenshot in the image list to validate.
-    imagesWithoutSupabase = imageOptions.filter(
-      (fileUri) => !fileUri.startsWith(process.env.SUPABASE_URL as string),
-    );
-  }
+  const imagesWithoutProtected = imageOptions.filter(
+    (fileUri) =>
+      (!process.env.SUPABASE_URL ||
+        !fileUri.startsWith(process.env.SUPABASE_URL)) &&
+      !fileUri.startsWith(YOUTUBE_THUMBNAIL_URL),
+  );
 
-  if (imagesWithoutSupabase.length === 0) {
+  if (imagesWithoutProtected.length === 0) {
     return {
       imageOptions,
     };
   }
 
   // Split images into chunks of 10
-  const imageChunks = chunkArray(imagesWithoutSupabase, 10);
+  const imageChunks = chunkArray(imagesWithoutProtected, 10);
   let allIrrelevantIndices: number[] = [];
   let baseIndex = 0;
 
@@ -99,34 +100,54 @@ export async function validateImages(
     if (!imageMessages.length) {
       continue;
     }
-    const response = await model.invoke([
-      {
-        role: "system",
-        content: formattedSystemPrompt,
-      },
-      {
-        role: "user",
-        content: imageMessages,
-      },
-    ]);
-
-    const chunkAnalysis = parseResult(response.content as string);
-    // Convert chunk indices to global indices and add to our list of relevant indices
-    const globalIndices = chunkAnalysis.map((index) => index + baseIndex);
-    allIrrelevantIndices = [...allIrrelevantIndices, ...globalIndices];
+
+    try {
+      const response = await model.invoke([
+        {
+          role: "system",
+          content: formattedSystemPrompt,
+        },
+        {
+          role: "user",
+          content: imageMessages,
+        },
+      ]);
+
+      const chunkAnalysis = parseResult(response.content as string);
+      // Convert chunk indices to global indices and add to our list of relevant indices
+      const globalIndices = chunkAnalysis.map((index) => index + baseIndex);
+      allIrrelevantIndices = [...allIrrelevantIndices, ...globalIndices];
+    } catch (error) {
+      console.error(
+        `Failed to validate images.\nImage URLs: ${imageMessages
+          .filter((m) => m.fileUri)
+          .map((m) => m.fileUri)
+          .join(", ")}\n\nError:`,
+        error,
+      );
+      // Add all indices from the failed chunk to allIrrelevantIndices
+      const failedChunkIndices = Array.from(
+        { length: imageChunk.length },
+        (_, i) => i + baseIndex,
+      );
+      allIrrelevantIndices = [...allIrrelevantIndices, ...failedChunkIndices];
+    }
 
     baseIndex += imageChunk.length;
   }
 
-  const supabaseUrl = imageOptions.find((fileUri) =>
-    fileUri.startsWith(process.env.SUPABASE_URL as string),
+  const protectedUrls = imageOptions.filter(
+    (fileUri) =>
+      (process.env.SUPABASE_URL &&
+        fileUri.startsWith(process.env.SUPABASE_URL)) ||
+      fileUri.startsWith(YOUTUBE_THUMBNAIL_URL),
   );
 
   // Keep only the relevant images (those whose indices are in allIrrelevantIndices)
   return {
     imageOptions: [
-      ...(supabaseUrl ? [supabaseUrl] : []),
-      ...imagesWithoutSupabase.filter((_, index) =>
+      ...protectedUrls,
+      ...imagesWithoutProtected.filter((_, index) =>
         allIrrelevantIndices.includes(index),
       ),
     ],

src/agents/shared/nodes/verify-youtube.ts

@@ -146,7 +146,7 @@ export async function verifyYouTubeContent(
     return {
       relevantLinks: [state.link],
       pageContents: [videoSummary as string],
-      ...(videoThumbnail ? { imageUrls: [videoThumbnail] } : {}),
+      ...(videoThumbnail ? { imageOptions: [videoThumbnail] } : {}),
     };
   }
 

src/agents/verify-links/verify-links-state.ts

@@ -32,4 +32,11 @@ export const VerifyLinksGraphAnnotation = Annotation.Root({
     },
     default: () => [],
   }),
+  /**
+   * Image options to provide to the user.
+   */
+  imageOptions: Annotation<string[]>({
+    reducer: (_state, update) => update,
+    default: () => [],
+  }),
 });