From 2c4496b1294a8252efa5a248eb24c355a1e43a21 Mon Sep 17 00:00:00 2001 From: Xingyao Wang Date: Sun, 30 Mar 2025 14:34:07 -0700 Subject: [PATCH] feat: Use LLM-generated natural-language descriptions as conversation title (#7049) Co-authored-by: openhands --- .openhands/microagents/repo.md | 10 ++- .../chat/action-suggestions.test.tsx | 12 +-- openhands/events/observation/agent.py | 2 +- .../runtime/plugins/vscode/settings.json | 2 +- .../server/routes/manage_conversations.py | 61 ++++++++++++-- openhands/utils/conversation_summary.py | 57 +++++++++++++ tests/unit/test_conversation_summary.py | 83 +++++++++++++++++++ 7 files changed, 211 insertions(+), 16 deletions(-) create mode 100644 openhands/utils/conversation_summary.py create mode 100644 tests/unit/test_conversation_summary.py diff --git a/.openhands/microagents/repo.md b/.openhands/microagents/repo.md index 8c6640dbcb..cb026df28e 100644 --- a/.openhands/microagents/repo.md +++ b/.openhands/microagents/repo.md @@ -10,13 +10,19 @@ This repository contains the code for OpenHands, an automated AI software engine To set up the entire repo, including frontend and backend, run `make build`. You don't need to do this unless the user asks you to, or if you're trying to run the entire application. -Before pushing any changes, you should ensure that any lint errors or simple test errors have been fixed. +Before pushing any changes, you MUST ensure that any lint errors or simple test errors have been fixed. * If you've made changes to the backend, you should run `pre-commit run --all-files --config ./dev_config/python/.pre-commit-config.yaml` * If you've made changes to the frontend, you should run `cd frontend && npm run lint:fix && npm run build ; cd ..` +The pre-commit hooks MUST pass successfully before pushing any changes to the repository. This is a mandatory requirement to maintain code quality and consistency. + If either command fails, it may have automatically fixed some issues. You should fix any issues that weren't automatically fixed, -then re-run the command to ensure it passes. +then re-run the command to ensure it passes. Common issues include: +- Mypy type errors +- Ruff formatting issues +- Trailing whitespace +- Missing newlines at end of files ## Repository Structure Backend: diff --git a/frontend/__tests__/components/chat/action-suggestions.test.tsx b/frontend/__tests__/components/chat/action-suggestions.test.tsx index 7ae87fa1ae..ccc2eb1e81 100644 --- a/frontend/__tests__/components/chat/action-suggestions.test.tsx +++ b/frontend/__tests__/components/chat/action-suggestions.test.tsx @@ -22,11 +22,11 @@ vi.mock("#/context/auth-context", () => ({ describe("ActionSuggestions", () => { // Setup mocks for each test vi.clearAllMocks(); - + (useAuth as any).mockReturnValue({ githubTokenIsSet: true, }); - + (useSelector as any).mockReturnValue({ selectedRepository: "test-repo", }); @@ -66,16 +66,16 @@ describe("ActionSuggestions", () => { it("should have different prompts for 'Push to Branch' and 'Push & Create PR' buttons", () => { // This test verifies that the prompts are different in the component const component = render( {}} />); - + // Get the component instance to access the internal values const pushBranchPrompt = "Please push the changes to a remote branch on GitHub, but do NOT create a pull request. Please use the exact SAME branch name as the one you are currently on."; const createPRPrompt = "Please push the changes to GitHub and open a pull request. Please create a meaningful branch name that describes the changes."; - + // Verify the prompts are different expect(pushBranchPrompt).not.toEqual(createPRPrompt); - + // Verify the PR prompt mentions creating a meaningful branch name expect(createPRPrompt).toContain("meaningful branch name"); expect(createPRPrompt).not.toContain("SAME branch name"); }); -}); \ No newline at end of file +}); diff --git a/openhands/events/observation/agent.py b/openhands/events/observation/agent.py index 16686d3b13..a91bfba22e 100644 --- a/openhands/events/observation/agent.py +++ b/openhands/events/observation/agent.py @@ -113,7 +113,7 @@ class RecallObservation(Observation): f'repo_instructions={self.repo_instructions[:20]}...', f'runtime_hosts={self.runtime_hosts}', f'additional_agent_instructions={self.additional_agent_instructions[:20]}...', - f'date={self.date}' + f'date={self.date}', ] ) else: diff --git a/openhands/runtime/plugins/vscode/settings.json b/openhands/runtime/plugins/vscode/settings.json index 77c19483b7..63eeeb1a73 100644 --- a/openhands/runtime/plugins/vscode/settings.json +++ b/openhands/runtime/plugins/vscode/settings.json @@ -1,4 +1,4 @@ { "workbench.colorTheme": "Default Dark Modern", "workbench.startupEditor": "none" -} \ No newline at end of file +} diff --git a/openhands/server/routes/manage_conversations.py b/openhands/server/routes/manage_conversations.py index 453d686594..acc0ab92d9 100644 --- a/openhands/server/routes/manage_conversations.py +++ b/openhands/server/routes/manage_conversations.py @@ -243,6 +243,24 @@ async def get_conversation( try: metadata = await conversation_store.get_metadata(conversation_id) is_running = await conversation_manager.is_agent_loop_running(conversation_id) + + # Check if we need to update the title + if is_running and metadata: + # Check if the title is a default title (contains the conversation ID) + if metadata.title and conversation_id[:5] in metadata.title: + # Generate a new title + new_title = await auto_generate_title( + conversation_id, get_user_id(request) + ) + + if new_title: + # Update the metadata + metadata.title = new_title + await conversation_store.save_metadata(metadata) + + # Refresh metadata after update + metadata = await conversation_store.get_metadata(conversation_id) + conversation_info = await _get_conversation_info(metadata, is_running) return conversation_info except FileNotFoundError: @@ -265,6 +283,7 @@ def get_default_conversation_title(conversation_id: str) -> str: async def auto_generate_title(conversation_id: str, user_id: str | None) -> str: """ Auto-generate a title for a conversation based on the first user message. + Uses LLM-based title generation if available, otherwise falls back to a simple truncation. Args: conversation_id: The ID of the conversation @@ -292,11 +311,39 @@ async def auto_generate_title(conversation_id: str, user_id: str | None) -> str: break if first_user_message: + # Try LLM-based title generation first + from openhands.core.config.llm_config import LLMConfig + from openhands.utils.conversation_summary import generate_conversation_title + + # Get LLM config from user settings + try: + settings_store = await SettingsStoreImpl.get_instance(config, user_id) + settings = await settings_store.load() + + if settings and settings.llm_model: + # Create LLM config from settings + llm_config = LLMConfig( + model=settings.llm_model, + api_key=settings.llm_api_key, + base_url=settings.llm_base_url, + ) + + # Try to generate title using LLM + llm_title = await generate_conversation_title( + first_user_message, llm_config + ) + if llm_title: + logger.info(f'Generated title using LLM: {llm_title}') + return llm_title + except Exception as e: + logger.error(f'Error using LLM for title generation: {e}') + + # Fall back to simple truncation if LLM generation fails or is unavailable first_user_message = first_user_message.strip() title = first_user_message[:30] if len(first_user_message) > 30: title += '...' - logger.info(f'Generated title: {title}') + logger.info(f'Generated title using truncation: {title}') return title except Exception as e: logger.error(f'Error generating title: {str(e)}') @@ -315,10 +362,12 @@ async def update_conversation( if not metadata: return False - # If title is empty or unspecified, auto-generate it from the first user message + # If title is empty or unspecified, auto-generate it if not title or title.isspace(): title = await auto_generate_title(conversation_id, user_id) - if not title: + + # If we still don't have a title, use the default + if not title or title.isspace(): title = get_default_conversation_title(conversation_id) metadata.title = title @@ -361,9 +410,9 @@ async def _get_conversation_info( last_updated_at=conversation.last_updated_at, created_at=conversation.created_at, selected_repository=conversation.selected_repository, - status=ConversationStatus.RUNNING - if is_running - else ConversationStatus.STOPPED, + status=( + ConversationStatus.RUNNING if is_running else ConversationStatus.STOPPED + ), ) except Exception as e: logger.error( diff --git a/openhands/utils/conversation_summary.py b/openhands/utils/conversation_summary.py new file mode 100644 index 0000000000..5ccf620468 --- /dev/null +++ b/openhands/utils/conversation_summary.py @@ -0,0 +1,57 @@ +"""Utility functions for generating conversation summaries.""" + +from typing import Optional + +from openhands.core.config import LLMConfig +from openhands.core.logger import openhands_logger as logger +from openhands.llm.llm import LLM + + +async def generate_conversation_title( + message: str, llm_config: LLMConfig, max_length: int = 50 +) -> Optional[str]: + """Generate a concise title for a conversation based on the first user message. + + Args: + message: The first user message in the conversation. + llm_config: The LLM configuration to use for generating the title. + max_length: The maximum length of the generated title. + + Returns: + A concise title for the conversation, or None if generation fails. + """ + if not message or message.strip() == '': + return None + + # Truncate very long messages to avoid excessive token usage + if len(message) > 1000: + truncated_message = message[:1000] + '...(truncated)' + else: + truncated_message = message + + try: + llm = LLM(llm_config) + + # Create a simple prompt for the LLM to generate a title + messages = [ + { + 'role': 'system', + 'content': 'You are a helpful assistant that generates concise, descriptive titles for conversations with OpenHands. OpenHands is a helpful AI agent that can interact with a computer to solve tasks using bash terminal, file editor, and browser. Given a user message (which may be truncated), generate a concise, descriptive title for the conversation. Return only the title, with no additional text, quotes, or explanations.', + }, + { + 'role': 'user', + 'content': f'Generate a title (maximum {max_length} characters) for a conversation that starts with this message:\n\n{truncated_message}', + }, + ] + + response = llm.completion(messages=messages) + title = response.choices[0].message.content.strip() + + # Ensure the title isn't too long + if len(title) > max_length: + title = title[: max_length - 3] + '...' + + return title + except Exception as e: + logger.error(f'Error generating conversation title: {e}') + return None diff --git a/tests/unit/test_conversation_summary.py b/tests/unit/test_conversation_summary.py new file mode 100644 index 0000000000..b688b14fd4 --- /dev/null +++ b/tests/unit/test_conversation_summary.py @@ -0,0 +1,83 @@ +"""Tests for the conversation summary generator.""" + +from unittest.mock import MagicMock, patch + +import pytest + +from openhands.core.config import LLMConfig +from openhands.utils.conversation_summary import generate_conversation_title + + +@pytest.mark.asyncio +async def test_generate_conversation_title_empty_message(): + """Test that an empty message returns None.""" + result = await generate_conversation_title('', MagicMock()) + assert result is None + + result = await generate_conversation_title(' ', MagicMock()) + assert result is None + + +@pytest.mark.asyncio +async def test_generate_conversation_title_success(): + """Test successful title generation.""" + # Create a proper mock response + mock_response = MagicMock() + mock_response.choices = [MagicMock()] + mock_response.choices[0].message.content = 'Generated Title' + + # Create a mock LLM instance with a synchronous completion method + mock_llm = MagicMock() + mock_llm.completion = MagicMock(return_value=mock_response) + + # Patch the LLM class to return our mock + with patch('openhands.utils.conversation_summary.LLM', return_value=mock_llm): + result = await generate_conversation_title( + 'Can you help me with Python?', LLMConfig(model='test-model') + ) + + assert result == 'Generated Title' + # Verify the mock was called with the expected arguments + mock_llm.completion.assert_called_once() + + +@pytest.mark.asyncio +async def test_generate_conversation_title_long_title(): + """Test that long titles are truncated.""" + # Create a proper mock response with a long title + mock_response = MagicMock() + mock_response.choices = [MagicMock()] + mock_response.choices[ + 0 + ].message.content = 'This is a very long title that should be truncated because it exceeds the maximum length' + + # Create a mock LLM instance with a synchronous completion method + mock_llm = MagicMock() + mock_llm.completion = MagicMock(return_value=mock_response) + + # Patch the LLM class to return our mock + with patch('openhands.utils.conversation_summary.LLM', return_value=mock_llm): + result = await generate_conversation_title( + 'Can you help me with Python?', LLMConfig(model='test-model'), max_length=30 + ) + + # Verify the title is truncated correctly + assert len(result) <= 30 + assert result.endswith('...') + + +@pytest.mark.asyncio +async def test_generate_conversation_title_exception(): + """Test that exceptions are handled gracefully.""" + # Create a mock LLM instance with a synchronous completion method that raises an exception + mock_llm = MagicMock() + mock_llm.completion = MagicMock(side_effect=Exception('Test error')) + + # Patch the LLM class to return our mock + with patch('openhands.utils.conversation_summary.LLM', return_value=mock_llm): + result = await generate_conversation_title( + 'Can you help me with Python?', LLMConfig(model='test-model') + ) + + # Verify that None is returned when an exception occurs + assert result is None