add url info in browsing observation (#3815)

* add url info in browsing observation

* fix integration tests for url

---------

Co-authored-by: Engel Nyst <enyst@users.noreply.github.com>

agenthub/browsing_agent/browsing_agent.py

@@ -65,10 +65,15 @@ In order to accomplish my goal I need to send the information asked back to the
 """
 
 
-def get_prompt(error_prefix: str, cur_axtree_txt: str, prev_action_str: str) -> str:
+def get_prompt(
+    error_prefix: str, cur_url: str, cur_axtree_txt: str, prev_action_str: str
+) -> str:
     prompt = f"""\
 {error_prefix}
 
+# Current Page URL:
+{cur_url}
+
 # Current Accessibility Tree:
 {cur_axtree_txt}
 
@@ -139,6 +144,7 @@ class BrowsingAgent(Agent):
         """
         messages: list[Message] = []
         prev_actions = []
+        cur_url = ''
         cur_axtree_txt = ''
         error_prefix = ''
         last_obs = None
@@ -179,6 +185,9 @@ class BrowsingAgent(Agent):
                 self.error_accumulator += 1
                 if self.error_accumulator > 5:
                     return MessageAction('Too many errors encountered. Task failed.')
+
+            cur_url = last_obs.url
+
             try:
                 cur_axtree_txt = flatten_axtree_to_str(
                     last_obs.axtree_object,
@@ -204,7 +213,7 @@ class BrowsingAgent(Agent):
 
         messages.append(Message(role='system', content=[TextContent(text=system_msg)]))
 
-        prompt = get_prompt(error_prefix, cur_axtree_txt, prev_action_str)
+        prompt = get_prompt(error_prefix, cur_url, cur_axtree_txt, prev_action_str)
         messages.append(Message(role='user', content=[TextContent(text=prompt)]))
 
         flat_messages = self.llm.format_messages_for_llm(messages)

tests/integration/mock/eventstream_runtime/BrowsingAgent/test_browse_internet/prompt_001.log

@@ -105,8 +105,12 @@ click('a51')
 click('48', button='middle', modifiers=['Shift'])
 
 
+
 ----------
 
+# Current Page URL:
+
+
 # Current Accessibility Tree:
 
 

tests/integration/mock/eventstream_runtime/BrowsingAgent/test_browse_internet/prompt_002.log

@@ -105,8 +105,12 @@ click('a51')
 click('48', button='middle', modifiers=['Shift'])
 
 
+
 ----------
 
+# Current Page URL:
+http://localhost:8000/
+
 # Current Accessibility Tree:
 RootWebArea 'The Ultimate Answer', focused
 	[8] heading 'The Ultimate Answer'

tests/integration/mock/eventstream_runtime/BrowsingAgent/test_browse_internet/prompt_003.log

@@ -105,8 +105,12 @@ click('a51')
 click('48', button='middle', modifiers=['Shift'])
 
 
+
 ----------
 
+# Current Page URL:
+http://localhost:8000/
+
 # Current Accessibility Tree:
 RootWebArea 'The Ultimate Answer', focused
 	[8] heading 'The Ultimate Answer'

tests/integration/mock/eventstream_runtime/CodeActAgent/test_browse_internet/prompt_002.log

@@ -105,8 +105,12 @@ click('a51')
 click('48', button='middle', modifiers=['Shift'])
 
 
+
 ----------
 
+# Current Page URL:
+
+
 # Current Accessibility Tree:
 
 

tests/integration/mock/eventstream_runtime/CodeActAgent/test_browse_internet/prompt_003.log

@@ -105,8 +105,12 @@ click('a51')
 click('48', button='middle', modifiers=['Shift'])
 
 
+
 ----------
 
+# Current Page URL:
+http://localhost:8000/
+
 # Current Accessibility Tree:
 RootWebArea 'The Ultimate Answer', focused
 	[8] heading 'The Ultimate Answer'

tests/integration/mock/eventstream_runtime/CodeActAgent/test_browse_internet/prompt_004.log

@@ -105,8 +105,12 @@ click('a51')
 click('48', button='middle', modifiers=['Shift'])
 
 
+
 ----------
 
+# Current Page URL:
+http://localhost:8000/
+
 # Current Accessibility Tree:
 RootWebArea 'The Ultimate Answer', focused
 	[8] heading 'The Ultimate Answer'