update record

src/core/conversation_manager.py

@@ -7,7 +7,26 @@ from PyQt6.QtCore import QObject, QThread, QTimer
 from src.core.few_shot_agent import FewShotGenerateAgent
 from src.core.input_listener import InputListener
 from xbrain.core.chat import run
+import multiprocessing
+from multiprocessing import Process, Queue
 
+def run_agent_analysis_process(action_data):
+    """
+    独立的进程函数，运行在单独的进程中
+    """
+    try:
+        from src.core.few_shot_agent import FewShotGenerateAgent
+        agent = FewShotGenerateAgent()
+        result = agent(action_data)
+        return {
+            'step_number': action_data['step_number'],
+            'analysis': result
+        }
+    except Exception as e:
+        return {
+            'step_number': action_data['step_number'],
+            'analysis': f"Error analyzing step: {str(e)}"
+        }
 
 class ConversationManager(QObject):
     """
@@ -31,6 +50,11 @@ class ConversationManager(QObject):
         self.is_recording = False
         self.text_buffer = ""
         self.last_keypress_time = 0
+        self.step_counter = 0  # Add step counter
+        self.agent_queue = Queue()
+        self.agent_process = None
+        self.analysis_results = {}  # Store analysis results by step number
+        self.pool = multiprocessing.Pool(processes=1)  # 使用进程池
         
         # Start the conversation
         self.start_conversation()
@@ -86,28 +110,32 @@ class ConversationManager(QObject):
             self.chat_area.add_message("Xiao Hong", response)
     
     def analyze_action(self, action):
-        """
-        Analyze user actions during demonstration
+        """Analyze user actions during demonstration"""
+        self.step_counter += 1
         
-        Args:
-            action: Dict containing action data
-        """
+        # 准备简化的动作数据
+        action_data = {
+            'type': action['type'],
+            'event': str(action['event']),
+            'step_number': self.step_counter
+        }
+        
+        if action['type'] == 'keyboard' and self.text_buffer:
+            action_data['text_buffer'] = self.text_buffer
+            
+        # 记录动作
+        action['step_number'] = self.step_counter
         self.task_demonstration.append(action)
         
-        # Initialize status text
-        status_text = f"Action detected: {action}"
-        few_shot_agent = FewShotGenerateAgent()
-        # Format display based on action type
+        # 状态文本
+        status_text = f"Step {self.step_counter}: "
         if action["type"] == "mouse":
             self.text_buffer = ""
-            status_text = few_shot_agent(action)
-
+            status_text += f"Mouse action: {action['event']}"
         elif action["type"] == "keyboard":
             current_time = time.time()
-            
             # Process keyboard input
             key_str = str(action["event"])
-            
             # Handle printable characters
             if len(key_str) == 3 and key_str.startswith("'") and key_str.endswith("'"):
                 self.text_buffer += key_str[1]
@@ -116,26 +144,53 @@ class ConversationManager(QObject):
             elif "key.space" in key_str.lower():
                 self.text_buffer += " "
             elif "key.enter" in key_str.lower() or "return" in key_str.lower():
-                # status_text = f"Keyboard input completed: \"{self.text_buffer}\""
-                status_text = few_shot_agent(action)
+                status_text = f"Keyboard input completed: \"{self.text_buffer}\""
                 self.update_mini_window_status(status_text)
                 self.text_buffer = ""
-                return
+                
+                # 为键盘输入添加文本缓冲
+                action_data['text_buffer'] = self.text_buffer
+            
             elif "key.backspace" in key_str.lower() and self.text_buffer:
                 self.text_buffer = self.text_buffer[:-1]
             
             # Display buffer if timeout occurred
             if current_time - self.last_keypress_time > 2.0 and self.text_buffer:
-                status_text = f"Keyboard input: \"{self.text_buffer}\""
-                status_text = few_shot_agent(action)
+                status_text += f"Keyboard input: \"{self.text_buffer}\""
+                
+                # 为键盘输入添加文本缓冲
+                action_data['text_buffer'] = self.text_buffer
+            
             else:
-                status_text = f"Keyboard action: {action['event']} (current input: \"{self.text_buffer}\")"
+                status_text += f"Keyboard action: {action['event']} (current input: \"{self.text_buffer}\")"
             
             self.last_keypress_time = current_time
         
-        # Update mini window status
+        # 异步提交分析任务
+        self.pool.apply_async(
+            run_agent_analysis_process, 
+            args=(action_data,),
+            callback=self._handle_analysis_result
+        )
+        
+        # 更新状态显示
         self.update_mini_window_status(status_text)
     
+    def _handle_analysis_result(self, result):
+        """处理分析结果的回调函数"""
+        if result and 'step_number' in result:
+            self.analysis_results[result['step_number']] = result.get('analysis', '')
+
+    def _get_combined_results(self):
+        """Combine all available results in step order"""
+        if not self.analysis_results:
+            return None
+            
+        combined_text = "Analysis summary:\n"
+        for step in sorted(self.analysis_results.keys()):
+            combined_text += f"Step {step}: {self.analysis_results[step]}\n"
+        return combined_text
+
     def update_mini_window_status(self, text):
         """
         Update the status text in the mini window
@@ -171,6 +226,10 @@ class ConversationManager(QObject):
     
     def finish_demonstration(self):
         """Complete the demonstration recording process"""
+        # 关闭进程池并等待所有任务完成
+        self.pool.close()
+        self.pool.join()
+        
         # Clean up
         self.keyboard_mouse_listen.stop_listen()
         
@@ -184,11 +243,20 @@ class ConversationManager(QObject):
         self.is_recording = False
         self.save_task_demonstration()
         
+        # 显示分析结果
+        combined_results = self._get_combined_results()
+        if combined_results:
+            self.chat_area.add_message("System", "Task Analysis Summary:")
+            self.chat_area.add_message("System", combined_results)
+        
+        # 重新初始化进程池
+        self.pool = multiprocessing.Pool(processes=1)
+        
         # Show summary
-        action_count = len(self.task_demonstration)
-        response = f"I've successfully learned this task! Recorded and analyzed {action_count} key actions. " + \
+        response = f"I've successfully learned this task! Recorded and analyzed {self.step_counter} steps. " + \
                   "Feel free to assign similar tasks to me in the future. 😊"
         self.chat_area.add_message("Xiao Hong", response)
+        self.step_counter = 0  # Reset step counter
         self.conversation_state = "ready"
     
     def handle_task_demonstration(self, message):
@@ -228,4 +296,10 @@ class ConversationManager(QObject):
                 json.dump(self.task_demonstration, f, ensure_ascii=False, indent=2)
             self.chat_area.add_message("System", "Task demonstration saved successfully")
         except Exception as e:
-            self.chat_area.add_message("System", f"Error saving task demonstration: {str(e)}") 
+            self.chat_area.add_message("System", f"Error saving task demonstration: {str(e)}")
+
+    def __del__(self):
+        """析构函数，确保进程池正确关闭"""
+        if hasattr(self, 'pool'):
+            self.pool.close()
+            self.pool.join()