Feat: Add editable address bar in browser tab (#26)

README.md

@@ -8,7 +8,7 @@ The easiest way to run Kevin is to [![Open in GitHub Codespaces](https://github.
   4) [Showed relevant error in UI](https://github.com/OpenDevin/OpenDevin/pull/2657) 🚨
   5) [Added Event History Condenser](https://github.com/OpenDevin/OpenDevin/pull/2937) 📜
   6) [Feat: Persist sandbox for Event Runtime](https://github.com/SmartManoj/Kevin/commit/2200b21dd01ecf3618d7e676cf16f875c5fce154) 🥳🥳
-  7) [Parsed pip output and restarted kernel automatically (for bash too)](https://github.com/SmartManoj/Kevin/commit/c8a51c97f985a748761cc86bf6a36a8bac36a3e0) 📦
+  7) [Parsed pip output and restarted kernel automatically (for bash too)](https://github.com/SmartManoj/Kevin/commit/3b77d5b2ec592e0fcb5bd7ed8a0d5787378bc0de) 📦
 
 ### Bug Fixes:
   1) [Fixed GroqException - content must be a string for role system & assisstant](https://github.com/SmartManoj/Kevin/commit/30c98d458a299d789ebd6b8ada842c050bc91b20) 🛠️

agenthub/browsing_agent/browsing_agent.py

@@ -180,7 +180,9 @@ def step(self, state: State) -> Action:
                     return MessageAction('Too many errors encountered. Task failed.')
             cur_axtree_txt = last_obs.axtree_txt
             if cur_axtree_txt.startswith('AX Error:'):
-                return MessageAction('Error encountered when browsing.')
+                return MessageAction(
+                    f'Error encountered when browsing. {cur_axtree_txt}'
+                )
 
         goal, _ = state.get_current_user_intent()
 

agenthub/codeact_agent/codeact_agent.py

@@ -18,11 +18,13 @@
     IPythonRunCellAction,
     MessageAction,
 )
+from opendevin.events.action.browse import BrowseURLAction
 from opendevin.events.observation import (
     AgentDelegateObservation,
     CmdOutputObservation,
     IPythonRunCellObservation,
 )
+from opendevin.events.observation.browse import BrowserOutputObservation
 from opendevin.events.observation.observation import Observation
 from opendevin.events.serialization.event import truncate_content
 from opendevin.llm.llm import LLM
@@ -123,6 +125,8 @@ def action_to_str(self, action: Action) -> str:
             return f'{action.thought}\n<execute_browse>\n{action.inputs["task"]}\n</execute_browse>'
         elif isinstance(action, MessageAction):
             return action.content
+        elif isinstance(action, BrowseURLAction):
+            return f'Opening {action.url} in browser manually'
         elif isinstance(action, AgentSummarizeAction):
             return (
                 'Summary of all Action and Observations till now. \n'
@@ -139,6 +143,7 @@ def get_action_message(self, action: Action) -> Message | None:
             or isinstance(action, CmdRunAction)
             or isinstance(action, IPythonRunCellAction)
             or isinstance(action, MessageAction)
+            or isinstance(action, BrowseURLAction)
             or isinstance(action, AgentSummarizeAction)
             or (isinstance(action, AgentFinishAction) and action.source == 'agent')
         ):
@@ -191,6 +196,13 @@ def get_observation_message(self, obs: Observation) -> Message | None:
                 content=[TextContent(text=text)],
                 event_id=obs.id,
             )
+        elif isinstance(obs, BrowserOutputObservation):
+            text = 'OBSERVATION:\n' + truncate_content(obs.content, max_message_chars)
+            return Message(
+                role='user',
+                content=[TextContent(text=text)],
+                event_id=obs.id,
+            )
         return None
 
     def reset(self) -> None:

frontend/src/components/Browser.test.tsx

@@ -29,7 +29,7 @@ describe("Browser", () => {
       },
     });
 
-    expect(screen.getByText("https://example.com")).toBeInTheDocument();
+    expect(screen.getByRole("textbox")).toHaveValue("https://example.com");
     expect(screen.getByAltText(/browser screenshot/i)).toBeInTheDocument();
   });
 });

frontend/src/components/Browser.tsx

@@ -1,17 +1,29 @@
-import React from "react";
+import React, { useState } from "react";
 import { useTranslation } from "react-i18next";
 import { IoIosGlobe } from "react-icons/io";
 import { useSelector } from "react-redux";
 import { I18nKey } from "#/i18n/declaration";
 import { RootState } from "#/store";
+import { updateBrowserTabUrl } from "#/services/browseService";
 
 function Browser(): JSX.Element {
   const { t } = useTranslation();
-
   const { url, screenshotSrc } = useSelector(
     (state: RootState) => state.browser,
   );
 
+  const [editableUrl, setEditableUrl] = useState(url);
+
+  const handleUrlChange = (e: React.ChangeEvent<HTMLInputElement>) => {
+    setEditableUrl(e.target.value);
+  };
+
+  const handleURLBar = (e: React.KeyboardEvent<HTMLInputElement>) => {
+    if (e.key === "Enter") {
+      updateBrowserTabUrl(editableUrl);
+    }
+  };
+
   const imgSrc =
     screenshotSrc && screenshotSrc.startsWith("data:image/png;base64,")
       ? screenshotSrc
@@ -20,7 +32,13 @@ function Browser(): JSX.Element {
   return (
     <div className="h-full w-full flex flex-col text-neutral-400">
       <div className="w-full p-2 truncate border-b border-neutral-600">
-        {url}
+        <input
+          type="text"
+          value={editableUrl}
+          onChange={handleUrlChange}
+          onKeyDown={handleURLBar}
+          className="w-full bg-transparent border-none outline-none text-neutral-400"
+        />
       </div>
       <div className="overflow-y-auto grow scrollbar-hide rounded-xl">
         {screenshotSrc ? (

frontend/src/components/chat/ChatInterface.tsx

@@ -5,7 +5,7 @@ import { RiArrowRightDoubleLine } from "react-icons/ri";
 import { useTranslation } from "react-i18next";
 import { VscArrowDown } from "react-icons/vsc";
 import { FaRegThumbsDown, FaRegThumbsUp } from "react-icons/fa";
-import { useDisclosure, Tooltip } from "@nextui-org/react";
+import { useDisclosure } from "@nextui-org/react";
 import ChatInput from "./ChatInput";
 import Chat from "./Chat";
 import TypingIndicator from "./TypingIndicator";

frontend/src/components/terminal/Terminal.test.tsx

@@ -44,7 +44,9 @@ describe("Terminal", () => {
     expect(screen.getByText("Terminal")).toBeInTheDocument();
     expect(mockTerminal.open).toHaveBeenCalledTimes(1);
 
-    expect(mockTerminal.write).toHaveBeenCalledWith("$ ");
+    expect(mockTerminal.write).toHaveBeenCalledWith(
+      "opendevin@docker-desktop:/workspace $ ",
+    );
   });
 
   it("should load commands to the terminal", () => {
@@ -54,7 +56,7 @@ describe("Terminal", () => {
     ]);
 
     expect(mockTerminal.writeln).toHaveBeenNthCalledWith(1, "INPUT");
-    expect(mockTerminal.writeln).toHaveBeenNthCalledWith(2, "OUTPUT");
+    expect(mockTerminal.write).toHaveBeenNthCalledWith(2, "OUTPUT");
   });
 
   it("should write commands to the terminal", () => {
@@ -66,13 +68,13 @@ describe("Terminal", () => {
     });
 
     expect(mockTerminal.writeln).toHaveBeenNthCalledWith(1, "echo Hello");
-    expect(mockTerminal.writeln).toHaveBeenNthCalledWith(2, "Hello");
+    expect(mockTerminal.write).toHaveBeenNthCalledWith(2, "Hello");
 
     act(() => {
       store.dispatch(appendInput("echo World"));
     });
 
-    expect(mockTerminal.writeln).toHaveBeenNthCalledWith(3, "echo World");
+    expect(mockTerminal.writeln).toHaveBeenNthCalledWith(2, "echo World");
   });
 
   it("should load and write commands to the terminal", () => {
@@ -82,13 +84,13 @@ describe("Terminal", () => {
     ]);
 
     expect(mockTerminal.writeln).toHaveBeenNthCalledWith(1, "echo Hello");
-    expect(mockTerminal.writeln).toHaveBeenNthCalledWith(2, "Hello");
+    expect(mockTerminal.write).toHaveBeenNthCalledWith(2, "Hello");
 
     act(() => {
       store.dispatch(appendInput("echo Hello"));
     });
 
-    expect(mockTerminal.writeln).toHaveBeenNthCalledWith(3, "echo Hello");
+    expect(mockTerminal.writeln).toHaveBeenNthCalledWith(2, "echo Hello");
   });
 
   it("should end the line with a dollar sign after writing a command", () => {
@@ -99,7 +101,9 @@ describe("Terminal", () => {
     });
 
     expect(mockTerminal.writeln).toHaveBeenCalledWith("echo Hello");
-    expect(mockTerminal.write).toHaveBeenCalledWith("$ ");
+    expect(mockTerminal.write).toHaveBeenCalledWith(
+      "opendevin@docker-desktop:/workspace $ ",
+    );
   });
 
   // This test fails because it expects `disposeMock` to have been called before the component is unmounted.

frontend/src/hooks/useTerminal.ts

@@ -96,16 +96,12 @@ export const useTerminal = (commands: Command[] = []) => {
         const lines = command.content.split("\r\n");
 
         lines.forEach((line, index) => {
-          terminal.current?.write(line);
-          if (index < lines.length - 1) {
-            terminal.current?.write("\r\n");
+          if (index < lines.length - 1 || command.type === "input") {
+            terminal.current?.writeln(line);
+          } else {
+            terminal.current?.write(line);
           }
         });
-
-        if (command.type === "input") {
-          terminal.current.write("\r\n");
-        }
-
       }
 
       lastCommandIndex.current = commands.length; // Update the position of the last command

frontend/src/services/browseService.ts

@@ -0,0 +1,8 @@
+import ActionType from "#/types/ActionType";
+import Session from "./session";
+
+export function updateBrowserTabUrl(newUrl: string): void {
+  const event = { action: ActionType.BROWSE, args: { url: newUrl } };
+  const eventString = JSON.stringify(event);
+  Session.send(eventString);
+}

opendevin/controller/agent_controller.py

@@ -37,6 +37,7 @@
     ErrorObservation,
     Observation,
 )
+from opendevin.events.observation.browse import BrowserOutputObservation
 from opendevin.llm.llm import LLM
 
 # note: RESUME is only available on web GUI
@@ -200,6 +201,8 @@ async def on_event(self, event: Event):
                 logger.info(event, extra={'msg_type': 'OBSERVATION'})
             elif isinstance(event, CmdOutputObservation):
                 logger.info(event, extra={'msg_type': 'OBSERVATION'})
+            elif isinstance(event, BrowserOutputObservation):
+                logger.info(event, extra={'msg_type': 'OBSERVATION'})
             elif isinstance(event, AgentDelegateObservation):
                 self.state.history.on_event(event)
                 logger.info(event, extra={'msg_type': 'OBSERVATION'})

opendevin/runtime/browser/utils.py

@@ -31,7 +31,9 @@ async def browse(
         raise ValueError(f'Invalid action type: {action.action}')
 
     try:
-        # obs provided by BrowserGym: see https://github.com/ServiceNow/BrowserGym/blob/main/core/src/browsergym/core/env.py#L396
+        # obs provided by BrowserGym:
+        # https://github.com/ServiceNow/BrowserGym/blob/418421abdc5da4d77dc71d3b82a9e5e931be0c4f/browsergym/core/src/browsergym/core/env.py#L521
+        # https://github.com/ServiceNow/BrowserGym/blob/418421abdc5da4d77dc71d3b82a9e5e931be0c4f/browsergym/core/src/browsergym/core/env.py#L521
         obs = browser.step(action_str)
         try:
             axtree_txt = flatten_axtree_to_str(

opendevin/runtime/client/runtime.py

@@ -212,20 +212,6 @@ async def _ensure_session(self):
     )
     async def _wait_until_alive(self):
         logger.info('Reconnecting session')
-        container = self.docker_client.containers.get(self.container_name)
-        # print logs
-        _logs = container.logs(tail=10).decode('utf-8').split('\n')
-        # add indent
-        _logs = '\n'.join([f'    |{log}' for log in _logs])
-        logger.info(
-            '\n'
-            + '-' * 30
-            + 'Container logs (last 10 lines):'
-            + '-' * 30
-            + f'\n{_logs}'
-            + '\n'
-            + '-' * 90
-        )
         async with aiohttp.ClientSession() as session:
             async with session.get(f'{self.api_url}/alive') as response:
                 if response.status == 200:
@@ -263,7 +249,10 @@ async def close(self, close_client: bool = True):
         containers = self.docker_client.containers.list(all=True)
         for container in containers:
             try:
-                if container.name.startswith(self.container_name_prefix):
+                # only remove the container we created
+                # otherwise all other containers with the same prefix will be removed
+                # which will mess up with parallel evaluation
+                if container.name.startswith(self.container_name):
                     logs = container.logs(tail=1000).decode('utf-8')
                     logger.debug(
                         f'==== Container logs ====\n{logs}\n==== End of container logs ===='
@@ -301,7 +290,7 @@ async def run_action(self, action: Action) -> Observation:
             assert action.timeout is not None
 
             try:
-                logger.info('Executing command')
+                logger.info(f'Executing action {action}')
                 async with session.post(
                     f'{self.api_url}/execute_action',
                     json={'action': event_to_dict(action)},

opendevin/server/session/session.py

@@ -17,6 +17,7 @@
     CmdOutputObservation,
     NullObservation,
 )
+from opendevin.events.observation.browse import BrowserOutputObservation
 from opendevin.events.serialization import event_from_dict, event_to_dict
 from opendevin.events.stream import EventStreamSubscriber
 from opendevin.llm.llm import LLM
@@ -135,7 +136,7 @@ async def on_event(self, event: Event):
         if event.source == EventSource.AGENT:
             await self.send(event_to_dict(event))
         elif event.source == EventSource.USER and isinstance(
-            event, CmdOutputObservation
+            event, (CmdOutputObservation, BrowserOutputObservation)
         ):
             await self.send(event_to_dict(event))