📦 NEW: Crawler SDK support (#80)

* 📦 NEW: Crawl function in SDK

* 📦 NEW: Crawl example

* 👌 IMPROVE: Crawl response type

examples/nodejs/.env.example

@@ -4,3 +4,4 @@ LANGBASE_SDK_GENERATE_PIPE=""
 LANGBASE_SDK_CHAT_PIPE=""
 LANGBASE_API_KEY=""
 WEB_SEARCH_KEY=""
+CRAWL_KEY=""

examples/nodejs/examples/tools/crawl.ts

@@ -0,0 +1,18 @@
+import 'dotenv/config';
+import {Langbase} from 'langbase';
+
+const langbase = new Langbase({
+	apiKey: process.env.LANGBASE_API_KEY!,
+});
+
+async function main() {
+	const results = await langbase.tool.crawl({
+		url: ['https://langbase.com', 'https://langbase.com/about'],
+		max_pages: 1,
+		api_key: process.env.CRAWL_KEY,
+	});
+
+	console.log(results);
+}
+
+main();

examples/nodejs/examples/tools/web-search.ts

@@ -8,7 +8,7 @@ const langbase = new Langbase({
 async function main() {
 	const results = await langbase.tool.webSearch({
 		query: 'AI Engineer',
-		apiKey: process.env.WEB_SEARCH_KEY,
+		api_key: process.env.WEB_SEARCH_KEY,
 	});
 
 	console.log(results);

examples/nodejs/package.json

@@ -28,7 +28,8 @@
     "stream-text": "npx tsx ./examples/pipes/stream-text.ts",
     "stream-text-generate-pipe": "npx tsx ./examples/pipes/stream-text-generate-pipe.ts",
     "stream-text-chat-pipe": "npx tsx ./examples/pipes/stream-text-chat-pipe.ts",
-    "tools.web-search": "npx tsx ./examples/tools/web-search.ts"
+    "tools.web-search": "npx tsx ./examples/tools/web-search.ts",
+    "tools.crawl": "npx tsx ./examples/tools/crawl.ts"
   },
   "keywords": [],
   "author": "Ahmad Awais <[email protected]> (https://twitter.com/MrAhmadAwais)",

examples/nodejs/readme.md

@@ -32,4 +32,5 @@ npm run memory.docs.retry-embed
 
 # tools
 npm run tools.web-search
+npm run tools.crawl
 ```

packages/langbase/src/langbase/langbase.ts

@@ -252,14 +252,25 @@ export interface ToolWebSearchOptions {
 	query: string;
 	total_results?: number;
 	domains?: string[];
-	apiKey?: string;
+	api_key?: string;
 }
 
 export interface ToolWebSearchResponse {
 	url: string;
 	content: string;
 }
 
+export interface ToolCrawlOptions {
+	url: string[];
+	max_pages?: number;
+	api_key?: string;
+}
+
+export interface ToolCrawlResponse {
+	url: string;
+	content: string;
+}
+
 export class Langbase {
 	private request: Request;
 	private apiKey: string;
@@ -296,6 +307,7 @@ export class Langbase {
 	};
 
 	public tool: {
+		crawl: (options: ToolCrawlOptions) => Promise<ToolCrawlResponse[]>;
 		webSearch: (
 			options: ToolWebSearchOptions,
 		) => Promise<ToolWebSearchResponse[]>;
@@ -331,6 +343,7 @@ export class Langbase {
 		};
 
 		this.tool = {
+			crawl: this.webCrawl.bind(this),
 			webSearch: this.webSearch.bind(this),
 		};
 	}
@@ -559,12 +572,36 @@ export class Langbase {
 	private async webSearch(
 		options: ToolWebSearchOptions,
 	): Promise<ToolWebSearchResponse[]> {
+		const apiKey = options.api_key ? options.api_key : null;
+		apiKey && delete options.api_key;
 		return this.request.post({
 			endpoint: '/v1/tools/web-search',
 			body: options,
 			headers: {
-				...(options.apiKey && {
-					'LB-WEB-SEARCH-KEY': options.apiKey,
+				...(apiKey && {
+					'LB-WEB-SEARCH-KEY': apiKey,
+				}),
+			},
+		});
+	}
+
+	/**
+	 * Performs a web crawls on target websites using the Langbase API.
+	 *
+	 * @param options - Crawl configuration options
+	 * @returns An array of responses containing data from the crawl operation.
+	 */
+	private async webCrawl(
+		options: ToolCrawlOptions,
+	): Promise<ToolCrawlResponse[]> {
+		const apiKey = options.api_key ? options.api_key : null;
+		apiKey && delete options.api_key;
+		return this.request.post({
+			endpoint: '/v1/tools/crawl',
+			body: options,
+			headers: {
+				...(apiKey && {
+					'LB-CRAWL-KEY': apiKey,
 				}),
 			},
 		});