V1: Experiment manager (#11388)

Co-authored-by: openhands <openhands@all-hands.dev>

openhands/app_server/app_conversation/live_status_app_conversation_service.py

@@ -5,7 +5,7 @@ from dataclasses import dataclass
 from datetime import datetime, timedelta
 from time import time
 from typing import AsyncGenerator, Sequence
-from uuid import UUID
+from uuid import UUID, uuid4
 
 import httpx
 from fastapi import Request
@@ -52,6 +52,7 @@ from openhands.app_server.services.injector import InjectorState
 from openhands.app_server.services.jwt_service import JwtService
 from openhands.app_server.user.user_context import UserContext
 from openhands.app_server.utils.async_remote_workspace import AsyncRemoteWorkspace
+from openhands.experiments.experiment_manager import ExperimentManagerImpl
 from openhands.integrations.provider import ProviderType
 from openhands.sdk import LocalWorkspace
 from openhands.sdk.conversation.secret_source import LookupSecret, StaticSecret
@@ -458,10 +459,17 @@ class LiveStatusAppConversationService(GitAppConversationService):
             model=user.llm_model,
             base_url=user.llm_base_url,
             api_key=user.llm_api_key,
-            service_id='agent',
+            usage_id='agent',
         )
         agent = get_default_agent(llm=llm)
+
+        conversation_id = uuid4()
+        agent = ExperimentManagerImpl.run_agent_variant_tests__v1(
+            user.id, conversation_id, agent
+        )
+
         start_conversation_request = StartConversationRequest(
+            conversation_id=conversation_id,
             agent=agent,
             workspace=workspace,
             confirmation_policy=AlwaysConfirm()

openhands/experiments/experiment_manager.py

@@ -1,9 +1,11 @@
 import os
+from uuid import UUID
 
 from pydantic import BaseModel
 
 from openhands.core.config.openhands_config import OpenHandsConfig
 from openhands.core.logger import openhands_logger as logger
+from openhands.sdk import Agent
 from openhands.server.session.conversation_init_data import ConversationInitData
 from openhands.server.shared import file_store
 from openhands.storage.locations import get_experiment_config_filename
@@ -29,6 +31,12 @@ def load_experiment_config(conversation_id: str) -> ExperimentConfig | None:
 
 
 class ExperimentManager:
+    @staticmethod
+    def run_agent_variant_tests__v1(
+        user_id: str | None, conversation_id: UUID, agent: Agent
+    ) -> Agent:
+        return agent
+
     @staticmethod
     def run_conversation_variant_test(
         user_id: str | None,