feat: Evals

.eslintrc.cjs

@@ -23,8 +23,6 @@ module.exports = {
     "*.d.ts",
   ],
   rules: {
-    "no-process-env": 2,
-    "no-instanceof/no-instanceof": 2,
     "@typescript-eslint/explicit-module-boundary-types": 0,
     "@typescript-eslint/no-empty-function": 0,
     "@typescript-eslint/no-shadow": 0,
@@ -33,6 +31,7 @@ module.exports = {
     "@typescript-eslint/no-unused-vars": ["warn", { args: "none" }],
     "@typescript-eslint/no-floating-promises": "error",
     "@typescript-eslint/no-misused-promises": "error",
+    "@typescript-eslint/no-explicit-any": 0,
     camelcase: 0,
     "class-methods-use-this": 0,
     "import/extensions": [2, "ignorePackages"],

package.json

@@ -19,9 +19,14 @@
     "format:check": "prettier --check .",
     "lint:langgraph-json": "node scripts/checkLanggraphPaths.js",
     "lint:all": "yarn lint & yarn lint:langgraph-json & yarn format:check",
-    "test:all": "yarn test && yarn test:int && yarn lint:langgraph"
+    "test:all": "yarn test && yarn test:int && yarn lint:langgraph",
+    "run:eval:general": "yarn tsx src/evals/general/index.ts",
+    "run:eval:github": "yarn tsx src/evals/github/index.ts",
+    "run:eval:twitter": "yarn tsx src/evals/twitter/index.ts",
+    "run:eval:youtube": "yarn tsx src/evals/youtube/index.ts"
   },
   "dependencies": {
+    "@googleapis/youtube": "^20.0.0",
     "@langchain/anthropic": "^0.3.8",
     "@langchain/community": "^0.3.15",
     "@langchain/core": "^0.3.18",
@@ -30,13 +35,16 @@
     "@mendable/firecrawl-js": "0.0.36",
     "@slack/web-api": "^7.7.0",
     "cheerio": "^1.0.0",
+    "google-auth-library": "^9.15.0",
+    "langsmith": "^0.2.7",
     "moment": "^2.30.1",
     "twitter-api-v2": "^1.18.2",
     "zod": "^3.23.8"
   },
   "devDependencies": {
     "@eslint/eslintrc": "^3.1.0",
     "@eslint/js": "^9.9.1",
+    "@jest/globals": "^29.7.0",
     "@tsconfig/recommended": "^1.0.7",
     "@types/jest": "^29.5.0",
     "@typescript-eslint/eslint-plugin": "^5.59.8",
@@ -50,6 +58,7 @@
     "jest": "^29.7.0",
     "prettier": "^3.3.3",
     "ts-jest": "^29.1.0",
+    "tsx": "^4.19.2",
     "typescript": "^5.3.3"
   }
 }

src/agent/nodes/ingest-data.ts

@@ -27,7 +27,6 @@ export async function ingestData(
   const client = new SlackMessageFetcher({
     channelId: channelId,
   });
-  console.log("Before fetching messages");
   const recentMessages = await client.fetchLast24HoursMessages(
     config.configurable?.maxMessages,
   );
@@ -44,7 +43,7 @@ export async function ingestData(
       links,
     };
   });
-  console.log("returning", messagesWithUrls.length, " messages");
+
   return {
     slackMessages: messagesWithUrls,
   };

src/agent/subgraphs/generate-post/generate-post-state.ts

@@ -48,7 +48,7 @@ export const GraphAnnotation = Annotation.Root({
     default: () => [],
   }),
   /**
-   * The content of the Tweet/LinkedIn post.
+   * The generated posts for LinkedIn/Twitter. Contains an array of different posts to choose from.
    */
-  post: Annotation<string>,
+  posts: Annotation<string[]>,
 });

src/agent/subgraphs/generate-post/nodes/generate-content-report.ts

@@ -4,20 +4,37 @@ import { LANGCHAIN_PRODUCTS_CONTEXT } from "../prompts.js";
 import { ChatAnthropic } from "@langchain/anthropic";
 
 const GENERATE_REPORT_PROMPT = `You are a highly regarded marketing employee at LangChain.
-You have been tasked with writing a report summary on content submitted to you from a third party in hopes of having it promoted by LangChain.
-This summary report will then be used to craft Tweets and LinkedIn posts promoting the content and LangChain products.
-LangChain has a policy of promoting any content submitted that uses LangChain's products.
+You have been tasked with writing a marketing report on content submitted to you from a third party which uses LangChain's products.
+This marketing report will then be used to craft Tweets and LinkedIn posts promoting the content and LangChain products.
 
 Here is some context about the different LangChain products and services:
+<langchain-context>
 ${LANGCHAIN_PRODUCTS_CONTEXT}
+</langchain-context>
 
-Given this context, examine the users input closely, and generate a summary report on it.
-
-The summary report should follow the following structure guidelines:
+The marketing report should follow the following structure guidelines. It will be made up of three main sections outlined below:
 <structure guidelines>
-1. The first part of the report should be a high level overview of the content. Include the name, what it does/what it aims to achieve/the problems it solves.
-2. The second part should be all about how it implements LangChain's products/services. Cover what product(s) it uses. How these products are used, and why they're important to the application. This should be technical and detailed. Ensure you clearly state the LangChain product(s) used at the top of this section.
-3. The final part should go into detail covering anything the first two parts missed. This should be a detailed technical overview of the content, and interesting facts you found that readers might find engaging. This part does NOT need to long, and if you've already covered everything, you can skip it. Remember you do NOT want to bore the readers with repetitive information.
+<part key="1">
+This is the introduction and summary of the content. This must include key details such as:
+- the name of the content/product/service.
+- what the content/product/service does, and/or the problems it solves.
+- unique selling points or interesting facts about the content.
+- a high level summary of the content/product/service.
+</part>
+
+<part key="2">
+This section should focus on how the content implements LangChain's products/services. It should include:
+- the LangChain product(s) used in the content.
+- how these products are used in the content.
+- why these products are important to the application.
+</part>
+
+<part key="3">
+This section should cover any additional details about the content that the first two parts missed. It should include:
+- a detailed technical overview of the content.
+- interesting facts about the content.
+- any other relevant information that may be engaging to readers.
+</part>
 </structure guidelines>
 
 Follow these rules and guidelines when generating the report:
@@ -26,12 +43,40 @@ Follow these rules and guidelines when generating the report:
 - The final Tweet/LinkedIn post will be developer focused, so ensure the report is technical and detailed.
 - Include any relevant links found in the content in the report.
 - Include details about what the product does/what problem it solves.
-- Use proper markdown styling when formatting the report summary.
+- Use proper markdown styling when formatting the marketing report.
 - If possible, keep the post at or under 280 characters (not including the URL) for conciseness.
+- Generate the report in English, even if the content submitted is not in English.
 <rules>
 
+Lastly, you should use the following process when writing the report:
+<writing-process>
+- First, read over the content VERY thoroughly.
+- Take notes, and write down your thoughts about the content after reading it carefully. These should be interesting insights or facts which you think you'll need later on when writing the final report. This should be the first text you write. ALWAYS perform this step first, and wrap the notes and thoughts inside a "<thinking>" tag.
+- Finally, write the report. Use the notes and thoughts you wrote down in the previous step to help you write the report. This should be the second and last text you write. Wrap your report inside a "<report>" tag.
+</writing-process>
+
 Do not include any personal opinions or biases in the report. Stick to the facts and technical details.
-Your response should ONLY include the report summary, and no other text.`;
+Your response should ONLY include the marketing report, and no other text.
+
+Given these instructions, examine the users input closely, and generate a marketing report on it.`;
+
+/**
+ * Parse the LLM generation to extract the report from inside the <report> tag.
+ * If the report can not be parsed, the original generation is returned.
+ * @param generation The text generation to parse
+ * @returns The parsed generation, or the unmodified generation if it cannot be parsed
+ */
+function parseGeneration(generation: string): string {
+  const reportMatch = generation.match(/<report>([\s\S]*?)<\/report>/);
+  if (!reportMatch) {
+    console.warn(
+      "Could not parse report from generation:\nSTART OF GENERATION\n\n",
+      generation,
+      "\n\nEND OF GENERATION",
+    );
+  }
+  return reportMatch ? reportMatch[1].trim() : generation;
+}
 
 const formatReportPrompt = (pageContents: string[]): string => {
   return `The following text contains summaries, or entire pages from the content I submitted to you. Please review the content and generate a report on it.
@@ -61,6 +106,6 @@ export async function generateContentReport(
   ]);
 
   return {
-    report: result.content as string,
+    report: parseGeneration(result.content as string),
   };
 }

src/agent/subgraphs/generate-post/nodes/generate-post.ts

@@ -2,14 +2,7 @@ import { LangGraphRunnableConfig } from "@langchain/langgraph";
 import { GraphAnnotation } from "../generate-post-state.js";
 import { ChatAnthropic } from "@langchain/anthropic";
 
-const GENERATE_POST_PROMPT = `You are a highly regarded marketing employee at LangChain, working on crafting thoughtful and engaging content for LangChain's LinkedIn page.
-You've been provided with a report on some content that you need to turn into a LinkedIn post. This content & report have been submitted to you by a third party who is looking to get their content promoted by LangChain.
-
-You should use the given report to generate an engaging, professional, and informative LinkedIn post that will be used to promote the content and LangChain's products.
-
-The following are examples of LinkedIn posts on third party content that have done well, and you should use them as inspiration for your post:
-
-<example index="1">
+const TWEET_EXAMPLES = `<example index="1">
 Podcastfy.ai 🎙️🤖
 
 An Open Source API alternative to NotebookLM's podcast product
@@ -37,30 +30,126 @@ RepoGPT is an open-source, AI-powered assistant
 Chat with your repositories using natural language to get insights, generate documentation, or receive code suggestions
 
 https://repogpt.com
+</example>`;
+
+const TWEET_EXAMPLES_BULLET_POINT = `<example index="1">
+🌐 Build agents that can interact with any website
+
+Check out this video by 
+@DendriteSystems
+ showing how to build an agent that can interact with websites just like a human would!
+
+This video demonstrates a workflow that:
+
+- Finds competitors on Product Hunt and Hacker News
+- Drafts an email about new competitors
+- Sends the email via Outlook
+
+📺 Video: https://youtube.com/watch?v=BGvqeRB4Jpk
+🧠 Repo: https://github.com/dendrite-systems/dendrite-examples
 </example>
 
-Now that you've seen some examples, lets's cover the structure of the LinkedIn post you should follow:
-<structure instructions>
-1. Post header. This should be a very short header, no more than 5 words that describes the content. This should ideally include one to two emojis, the name of the content provided (if applicable), and the LangChain product(s) the content uses.
-2. Post body. This should be a two part body. The first part should be a concise, high level overview of the content and what it does, or aims to achieve. The second part should be a high level overview of how it uses LangChain's product(s) to do that. Ensure both of these are short, totally no more than 3 shorter sentences in total. Ensure these two parts are split with a newline between them.
-3. Call to action. This should be a short sentence that encourages the reader to click the link to the content being promoted. This should include the link to the content being promoted. Optionally, you can include an emoji here.
-</structure instructions>
+<example index="2">
+🖋️AgentWrite LangGraph
+
+Features
+- Automated content planning
+- Paragraph-by-paragraph content generation
+- Integration with multiple LLMs (OpenAI, GROQ, OLLaMA)
+- Flexible workflow management using LangGraph
+- Markdown output for generated content
 
-This structure should ALWAYS be followed. And remember, the shorter and more engaging the post, the better (your bonus depends on this!!).
+https://github.com/samwit/agent_tutorials/tree/main/agent_write
+</example>
 
-Here are a set of rules and guidelines you should strictly follow when creating the LinkedIn post:
+<example index="3">
+✈️AI Travel Agent
+
+This is one of the most comprehensive examples we've seen of a LangGraph agent. It's specifically designed to be a real world practical use case
+
+Features
+- Stateful Interactions
+- Human-in-the-Loop
+- Dynamic LLMs
+- Email Automation
+
+https://github.com/nirbar1985/ai-travel-agent
+</example>`;
+
+const CONTENT_SUMMARY_PROMPT = `This section will contain the main content of the post. The post body should contain a concise, high-level overview of the content/product/service outlines in the marketing report.
+It should focus on what the content does, or the problem it solves. Also include details on how the content implements LangChain's product(s) and why these products are important to the application.
+Ensure this is short, no more than 3 sentences. You should NOT make the main focus of this on LangChain, but instead on the content itself. Remember, the content/product/service outlined in the marketing report is the main focus of this post.`;
+
+const BULLET_POINT_PROMPT = `This section will contain the main content of the post. The post body should contain 2-5 bullet points that cover the main points of the content/product/service outlines in the marketing report.
+Each bullet point should have an emoji, and should be very concise, less than a single sentence. When thinking about what content to use for the bullet points, you should think from the point of view of a developer/CTO, and why they would benefit from the content.
+Using this thinking, carefully craft your bullet points. At least one of these bullet points should cover how it uses LangChain's product(s). Remember, the content/product/service outlined in the marketing report is the main focus of this post.`;
+
+const BASE_GENERATE_POST_PROMPT = ({
+  bulletPointStyle,
+}: {
+  bulletPointStyle: boolean;
+}) => `You're a highly regarded marketing employee at LangChain, working on crafting thoughtful and engaging content for LangChain's LinkedIn and Twitter pages.
+You've been provided with a report on some content that you need to turn into a LinkedIn/Twitter post. The same post will be used for both platforms.
+Your coworker has already taken the time to write a detailed marketing report on this content for you, so please take your time and read it carefully.
+
+The following are examples of LinkedIn/Twitter posts on third-party LangChain content that have done well, and you should use them as style inspiration for your post:
+<examples>
+${bulletPointStyle ? TWEET_EXAMPLES_BULLET_POINT : TWEET_EXAMPLES}
+</examples>
+
+Now that you've seen some examples, lets's cover the structure of the LinkedIn/Twitter post you should follow. The post should have three main sections, outlined below:
+<structure-instructions>
+<section key="1">
+The first part of the post is the header. This should be very short, no more than 5 words, and should include one to two emojis, and the name of the content provided. If the marketing report does not specify a name, you should get creative and come up with a catchy title for it.
+</section>
+
+<section key="2">
+${bulletPointStyle ? BULLET_POINT_PROMPT : CONTENT_SUMMARY_PROMPT}
+</section>
+
+<section key="3">
+The final section of the post should contain a call to action. This should be a short sentence that encourages the reader to click the link to the content being promoted. Optionally, you can include an emoji here.
+</section>
+</structure-instructions>
+
+This structure should ALWAYS be followed. And remember, the shorter and more engaging the post, the better (your yearly bonus depends on this!!).
+
+Here are a set of rules and guidelines you should strictly follow when creating the LinkedIn/Twitter post:
 <rules>
 - Focus your post on what the content covers, aims to achieve, and how it uses LangChain's product(s) to do that. This should be concise and high level.
-- Do not include technical details unless it is the entire focus of the content.
+- Do not make the post over technical as some of our audience may not be advanced developers, but ensure it is technical enough to engage developers.
 - Keep posts short, concise and engaging
-- Limit the use of emojis to the post header, and optionally in the call to action.
+- Limit the use of emojis to the post header, ${bulletPointStyle ? "optionally in the call to action, and optionally in the bullet points." : "and optionally in the call to action."}
 - NEVER use hashtags in the post.
-- Include the link to the content being promoted in the call to action section of the post.
+- ALWAYS include the link to the content being promoted in the call to action section of the post.
 </rules>
 
-Given these examples, rules, and the content provided by the user, curate a LinkedIn post that is engaging and follows the structure of the examples provided.
-
-Finally, ensure your response ONLY includes the LinkedIn post content, and does not include any additional information.`;
+Lastly, you should follow the process below when writing the LinkedIn/Twitter post:
+<writing-process>
+Step 1. First, read over the marketing report VERY thoroughly.
+Step 2. Take notes, and write down your thoughts about the report after reading it carefully. This should include details you think will help make the post more engaging, and your initial thoughts about what to focus the post on, the style, etc. This should be the first text you write. Wrap the notes and thoughts inside a "<thinking>" tag.
+Step 3. Lastly, write the LinkedIn/Twitter post. Use the notes and thoughts you wrote down in the previous step to help you write the post. This should be the last text you write. Wrap your report inside a "<post>" tag. Ensure you write only ONE post for both LinkedIn and Twitter.
+</writing-process>
+
+Given these examples, rules, and the content provided by the user, curate a LinkedIn/Twitter post that is engaging and follows the structure of the examples provided.`;
+
+/**
+ * Parse the LLM generation to extract the report from inside the <report> tag.
+ * If the report can not be parsed, the original generation is returned.
+ * @param generation The text generation to parse
+ * @returns The parsed generation, or the unmodified generation if it cannot be parsed
+ */
+function parseGeneration(generation: string): string {
+  const reportMatch = generation.match(/<post>([\s\S]*?)<\/post>/);
+  if (!reportMatch) {
+    console.warn(
+      "Could not parse post from generation:\nSTART OF POST GENERATION\n\n",
+      generation,
+      "\n\nEND OF POST GENERATION",
+    );
+  }
+  return reportMatch ? reportMatch[1].trim() : generation;
+}
 
 const formatPrompt = (report: string, link: string) => {
   return `Here is the report I wrote on the content I'd like promoted by LangChain:
@@ -86,23 +175,38 @@ export async function generatePosts(
   }
   const postModel = new ChatAnthropic({
     model: "claude-3-5-sonnet-20241022",
-    temperature: 0,
+    temperature: 0.5,
   });
 
   const prompt = formatPrompt(state.report, state.relevantLinks[0]);
 
-  const result = await postModel.invoke([
-    {
-      role: "system",
-      content: GENERATE_POST_PROMPT,
-    },
-    {
-      role: "user",
-      content: prompt,
-    },
+  const [summaryPost, bulletPointsPost] = await Promise.all([
+    postModel.invoke([
+      {
+        role: "system",
+        content: BASE_GENERATE_POST_PROMPT({ bulletPointStyle: false }),
+      },
+      {
+        role: "user",
+        content: prompt,
+      },
+    ]),
+    postModel.invoke([
+      {
+        role: "system",
+        content: BASE_GENERATE_POST_PROMPT({ bulletPointStyle: true }),
+      },
+      {
+        role: "user",
+        content: prompt,
+      },
+    ]),
   ]);
 
   return {
-    post: result.content as string,
+    posts: [
+      parseGeneration(summaryPost.content as string),
+      parseGeneration(bulletPointsPost.content as string),
+    ],
   };
 }