github/OpenHands
1
0
Fork 0
mirror of https://github.com/OpenHands/OpenHands.git synced 2025-12-26 05:48:36 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add playwright and show screenshoots on web browser (#547)

Browse Source 
* add playwright and show screenshoots on web browser

* fix lints

* fix lint issues

* fix lint issues

* fix lint issues

* fix lint issues

* fix lint issues
This commit is contained in:
universea 2024-04-02 18:46:01 -04:00 committed by GitHub
parent c3f95f049f
commit 5f29df088a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 95 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
agenthub/monologue_agent/agent.py
View File

@ -85,6 +85,8 @@ class MonologueAgent(Agent):
self.memory = LongTermMemory()
def _add_event(self, event: dict):
if "extras" in event and "screenshot" in event["extras"]:
del event["extras"]["screenshot"]
if 'args' in event and 'output' in event['args'] and len(event['args']['output']) > MAX_OUTPUT_LENGTH:
event['args']['output'] = event['args']['output'][:MAX_OUTPUT_LENGTH] + "..."
@ -114,7 +116,7 @@ class MonologueAgent(Agent):
elif output_type == "recall":
observation = AgentRecallObservation(content=thought, memories=[])
elif output_type == "browse":
observation = BrowserOutputObservation(content=thought, url="")
observation = BrowserOutputObservation(content=thought, url="", screenshot="")
self._add_event(observation.to_dict())
output_type = ""
else:

5
agenthub/planner_agent/prompt.py
View File

@ -139,7 +139,10 @@ def get_prompt(plan: Plan, history: List[Tuple[Action, Observation]]):
history_dicts.append(action.to_dict())
latest_action = action
if not isinstance(observation, NullObservation):
history_dicts.append(observation.to_dict())
observation_dict = observation.to_dict()
if "extras" in observation_dict and "screenshot" in observation_dict["extras"]:
del observation_dict["extras"]["screenshot"]
history_dicts.append(observation_dict)
history_str = json.dumps(history_dicts, indent=2)
hint = ""

26
frontend/package-lock.json generated
View File

@ -3099,7 +3099,6 @@
"version": "0.11.0",
"resolved": "https://registry.npmmirror.com/@pkgjs/parseargs/-/parseargs-0.11.0.tgz",
"integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==",
"dev": true,
"optional": true,
"engines": {
"node": ">=14"
@ -10209,6 +10208,12 @@
"node": ">=16 || 14 >=14.17"
}
},
"node_modules/monaco-editor": {
"version": "0.47.0",
"resolved": "https://registry.npmjs.org/monaco-editor/-/monaco-editor-0.47.0.tgz",
"integrity": "sha512-VabVvHvQ9QmMwXu4du008ZDuyLnHs9j7ThVFsiJoXSOQk18+LF89N4ADzPbFenm0W4V2bGHnFBztIRQTgBfxzw==",
"peer": true
},
"node_modules/ms": {
"version": "2.1.2",
"resolved": "https://registry.npmmirror.com/ms/-/ms-2.1.2.tgz",
@ -12961,6 +12966,12 @@
"node": ">=0.4"
}
},
"node_modules/xterm": {
"version": "5.3.0",
"resolved": "https://registry.npmjs.org/xterm/-/xterm-5.3.0.tgz",
"integrity": "sha512-8QqjlekLUFTrU6x7xck1MsPzPA571K5zNqWm0M0oroYEWVOptZ0+ubQSkQ3uxIEhcIHRujJy6emDWX4A7qyFzg==",
"peer": true
},
"node_modules/xterm-addon-fit": {
"version": "0.8.0",
"resolved": "https://registry.npmmirror.com/xterm-addon-fit/-/xterm-addon-fit-0.8.0.tgz",
@ -15190,7 +15201,6 @@
"version": "0.11.0",
"resolved": "https://registry.npmmirror.com/@pkgjs/parseargs/-/parseargs-0.11.0.tgz",
"integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==",
"dev": true,
"optional": true
},
"@pkgr/core": {
@ -20319,6 +20329,12 @@
"resolved": "https://registry.npmmirror.com/minipass/-/minipass-7.0.4.tgz",
"integrity": "sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ=="
},
"monaco-editor": {
"version": "0.47.0",
"resolved": "https://registry.npmjs.org/monaco-editor/-/monaco-editor-0.47.0.tgz",
"integrity": "sha512-VabVvHvQ9QmMwXu4du008ZDuyLnHs9j7ThVFsiJoXSOQk18+LF89N4ADzPbFenm0W4V2bGHnFBztIRQTgBfxzw==",
"peer": true
},
"ms": {
"version": "2.1.2",
"resolved": "https://registry.npmmirror.com/ms/-/ms-2.1.2.tgz",
@ -22149,6 +22165,12 @@
"resolved": "https://registry.npmmirror.com/xtend/-/xtend-4.0.2.tgz",
"integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ=="
},
"xterm": {
"version": "5.3.0",
"resolved": "https://registry.npmjs.org/xterm/-/xterm-5.3.0.tgz",
"integrity": "sha512-8QqjlekLUFTrU6x7xck1MsPzPA571K5zNqWm0M0oroYEWVOptZ0+ubQSkQ3uxIEhcIHRujJy6emDWX4A7qyFzg==",
"peer": true
},
"xterm-addon-fit": {
"version": "0.8.0",
"resolved": "https://registry.npmmirror.com/xterm-addon-fit/-/xterm-addon-fit-0.8.0.tgz",

19
frontend/src/components/Browser.tsx
View File

@ -3,12 +3,29 @@ import { useSelector } from "react-redux";
import { RootState } from "../store";
function Browser(): JSX.Element {
const url = useSelector((state: RootState) => state.browser.url);
const { url, screenshotSrc } = useSelector(
(state: RootState) => state.browser,
);
const imgSrc =
screenshotSrc && screenshotSrc.startsWith("data:image/png;base64,")
? screenshotSrc
: `data:image/png;base64,${screenshotSrc || ""}`;
return (
<div className="h-full m-2 bg-bg-workspace mockup-browser">
<div className="mockup-browser-toolbar">
<div className="input">{url}</div>
</div>
{screenshotSrc ? (
<img
src={imgSrc}
alt="Browser Screenshot"
style={{ maxWidth: "100%", height: "auto" }}
/>
) : (
<div>No screenshot available.</div>
)}
</div>
);
}

9
frontend/src/socket/observations.ts
View File

@ -1,7 +1,16 @@
import { appendAssistantMessage } from "../state/chatSlice";
import { setUrl, setScreenshotSrc } from "../state/browserSlice";
import store from "../store";
import { ObservationMessage } from "../types/Message";
export function handleObservationMessage(message: ObservationMessage) {
store.dispatch(appendAssistantMessage(message.message));
if (message.observation === "browse") {
if (message.extras?.screenshot) {
store.dispatch(setScreenshotSrc(message.extras.screenshot));
}
if (message.extras?.url) {
store.dispatch(setUrl(message.extras.url));
}
}
}

3
frontend/src/types/Message.tsx
View File

@ -21,4 +21,7 @@ export interface ObservationMessage {
// A friendly message that can be put in the chat log
message: string;
// optional screenshoot
screenshot?: string;
}

37
opendevin/action/browse.py
View File

@ -1,30 +1,45 @@
import requests
import base64
from dataclasses import dataclass
from opendevin.observation import BrowserOutputObservation
from typing import TYPE_CHECKING
from playwright.async_api import async_playwright
from .base import ExecutableAction
if TYPE_CHECKING:
from opendevin.controller import AgentController
@dataclass
class BrowseURLAction(ExecutableAction):
url: str
action: str = "browse"
def run(self, *args, **kwargs) -> BrowserOutputObservation:
async def run(self, controller: "AgentController") -> BrowserOutputObservation: # type: ignore
try:
response = requests.get(self.url)
return BrowserOutputObservation(
content=response.text,
status_code=response.status_code,
url=self.url
)
except requests.exceptions.RequestException as e:
async with async_playwright() as p:
browser = await p.chromium.launch()
page = await browser.new_page()
response = await page.goto(self.url)
# content = await page.content()
inner_text = await page.evaluate("() => document.body.innerText")
screenshot_bytes = await page.screenshot(full_page=True)
await browser.close()
screenshot_base64 = base64.b64encode(screenshot_bytes).decode("utf-8")
return BrowserOutputObservation(
content=inner_text, # HTML content of the page
screenshot=screenshot_base64, # Base64-encoded screenshot
url=self.url,
status_code=response.status if response else 0, # HTTP status code
)
except Exception as e:
return BrowserOutputObservation(
content=str(e),
screenshot="",
error=True,
url=self.url
)
@property
def message(self) -> str:
return f"Browsing URL: {self.url}"

9
opendevin/controller/agent_controller.py
View File

@ -1,7 +1,9 @@
import asyncio
import inspect
import traceback
from typing import List, Callable, Literal, Mapping, Any
from typing import List, Callable, Literal, Mapping, Awaitable, Any, cast
from termcolor import colored
from opendevin.plan import Plan
@ -144,7 +146,10 @@ class AgentController:
if action.executable:
try:
observation = action.run(self)
if inspect.isawaitable(action.run(self)):
observation = await cast(Awaitable[Observation], action.run(self))
else:
observation = action.run(self)
except Exception as e:
observation = AgentErrorObservation(str(e))
print_with_color(observation, "ERROR")

1
opendevin/observation/browse.py
View File

@ -9,6 +9,7 @@ class BrowserOutputObservation(Observation):
"""
url: str
screenshot: str
status_code: int = 200
error: bool = False
observation : str = "browse"