From b91a1dcc5cab656f7c727dcbd1d31b421ed240b2 Mon Sep 17 00:00:00 2001 From: yuruo Date: Tue, 25 Mar 2025 10:53:03 +0800 Subject: [PATCH] update project --- ui/chat_panel.py | 69 ++++++++++++++ ui/demonstration_panel.py | 48 ++++++++++ ui/main_window.py | 187 ++++++++++++++++---------------------- ui/recording_manager.py | 81 +++++++++++++++++ ui/recording_panel.py | 43 +++++++++ ui/settings_manager.py | 59 ++++++++++++ ui/task_panel.py | 30 ++++++ util/auto_control.py | 23 ++++- 8 files changed, 431 insertions(+), 109 deletions(-) create mode 100644 ui/chat_panel.py create mode 100644 ui/demonstration_panel.py create mode 100644 ui/recording_manager.py create mode 100644 ui/recording_panel.py create mode 100644 ui/settings_manager.py create mode 100644 ui/task_panel.py diff --git a/ui/chat_panel.py b/ui/chat_panel.py new file mode 100644 index 0000000..993dd60 --- /dev/null +++ b/ui/chat_panel.py @@ -0,0 +1,69 @@ +""" +Chat panel for autoMate +""" +from PyQt6.QtWidgets import QWidget, QVBoxLayout, QLabel, QTextEdit +from PyQt6.QtGui import QTextCursor, QTextCharFormat, QColor + +class ChatPanel(QWidget): + def __init__(self, parent=None): + super().__init__(parent) + self.setup_ui() + + def setup_ui(self): + """Initialize chat panel UI""" + chat_layout = QVBoxLayout(self) + chat_label = QLabel("Chat History") + self.chat_display = QTextEdit() + self.chat_display.setReadOnly(True) + chat_layout.addWidget(chat_label) + chat_layout.addWidget(self.chat_display) + + def update_chat(self, chatbox_messages): + """Update chat display with new messages""" + self.chat_display.clear() + + for msg in chatbox_messages: + role = msg["role"] + content = msg["content"] + + # Set different formats based on role + format = QTextCharFormat() + if role == "user": + format.setForeground(QColor(0, 0, 255)) # Blue for user + self.chat_display.append("You:") + else: + format.setForeground(QColor(0, 128, 0)) # Green for AI + self.chat_display.append("AI:") + + # Add content + cursor = self.chat_display.textCursor() + cursor.movePosition(QTextCursor.MoveOperation.End) + + # Special handling for HTML content + if "<" in content and ">" in content: + self.chat_display.insertHtml(content) + self.chat_display.append("") # Add empty line + else: + self.chat_display.append(content) + self.chat_display.append("") # Add empty line + + # Scroll to bottom + self.chat_display.verticalScrollBar().setValue( + self.chat_display.verticalScrollBar().maximum() + ) + + def append_message(self, message, color=None): + """Append a single message to chat display""" + if color: + self.chat_display.append(f"{message}") + else: + self.chat_display.append(message) + + # Scroll to bottom + self.chat_display.verticalScrollBar().setValue( + self.chat_display.verticalScrollBar().maximum() + ) + + def clear(self): + """Clear chat history""" + self.chat_display.clear() \ No newline at end of file diff --git a/ui/demonstration_panel.py b/ui/demonstration_panel.py new file mode 100644 index 0000000..e983fc0 --- /dev/null +++ b/ui/demonstration_panel.py @@ -0,0 +1,48 @@ +""" +Demonstration panel for autoMate +""" +from PyQt6.QtWidgets import QWidget, QHBoxLayout, QLabel, QPushButton, QApplication +from PyQt6.QtCore import Qt, QPoint + +class DemonstrationPanel(QWidget): + def __init__(self, parent=None, stop_callback=None): + super().__init__(parent, Qt.WindowType.WindowStaysOnTopHint | Qt.WindowType.FramelessWindowHint) + self.stop_callback = stop_callback + self.setup_ui() + self.position_to_bottom_right() + + def setup_ui(self): + demo_layout = QHBoxLayout() + self.setLayout(demo_layout) + + # autoMate logo + logo_label = QLabel("autoMate recording...") + logo_label.setStyleSheet("color: #4CAF50; font-weight: bold; font-size: 14px;") + demo_layout.addWidget(logo_label) + + # 停止按钮 + stop_demo_button = QPushButton("Stop") + stop_demo_button.setStyleSheet("background-color: #ff0000; color: white;") + stop_demo_button.clicked.connect(self.on_stop_clicked) + demo_layout.addWidget(stop_demo_button) + + demo_layout.addStretch() + + # 设置窗口样式 + self.setStyleSheet("background-color: #f0f0f0; border: 1px solid #999; padding: 8px;") + self.setFixedHeight(50) # 固定高度使其更紧凑 + self.resize(250, 50) + + def position_to_bottom_right(self): + screen = QApplication.primaryScreen() + screen_geometry = screen.availableGeometry() + window_geometry = self.frameGeometry() + position = QPoint( + screen_geometry.width() - window_geometry.width() - 20, + screen_geometry.height() - window_geometry.height() - 20 + ) + self.move(position) + + def on_stop_clicked(self): + if self.stop_callback: + self.stop_callback() \ No newline at end of file diff --git a/ui/main_window.py b/ui/main_window.py index 4968d49..d82770a 100644 --- a/ui/main_window.py +++ b/ui/main_window.py @@ -2,15 +2,15 @@ Main application window """ import os +import sys import keyboard from pathlib import Path from PyQt6.QtWidgets import (QMainWindow, QWidget, QVBoxLayout, QHBoxLayout, - QLabel, QLineEdit, QPushButton, QTableWidget, QTableWidgetItem, - QTextEdit, QSplitter, QMessageBox, QHeaderView, QDialog, QSystemTrayIcon) + QLabel, QLineEdit, QPushButton, QSplitter, QMessageBox, + QDialog, QSystemTrayIcon, QApplication) from PyQt6.QtCore import Qt, pyqtSlot, QSize -from PyQt6.QtGui import QPixmap, QIcon, QTextCursor, QTextCharFormat, QColor +from PyQt6.QtGui import QPixmap, QIcon, QKeySequence, QShortcut -from xbrain.utils.config import Config from auto_control.agent.vision_agent import VisionAgent from util.download_weights import OMNI_PARSER_DIR @@ -19,6 +19,10 @@ from ui.settings_dialog import SettingsDialog from ui.agent_worker import AgentWorker from ui.tray_icon import StatusTrayIcon from ui.hotkey_edit import DEFAULT_STOP_HOTKEY +from ui.task_panel import TaskPanel +from ui.chat_panel import ChatPanel +from ui.recording_manager import RecordingManager +from ui.settings_manager import SettingsManager # Intro text for application INTRO_TEXT = ''' @@ -32,6 +36,9 @@ class MainWindow(QMainWindow): super().__init__() self.args = args + # Initialize settings manager + self.settings_manager = SettingsManager() + # Initialize state self.state = self.setup_initial_state() @@ -40,6 +47,9 @@ class MainWindow(QMainWindow): yolo_model_path=os.path.join(OMNI_PARSER_DIR, "icon_detect", "model.pt") ) + # Initialize recording manager + self.recording_manager = RecordingManager(self) + # Setup UI and tray icon self.setup_tray_icon() self.setWindowTitle("autoMate") @@ -50,8 +60,6 @@ class MainWindow(QMainWindow): # Register hotkey handler self.hotkey_handler = None self.register_stop_hotkey() - - print("\n\n🚀 PyQt6 application launched") def setup_tray_icon(self): """Setup system tray icon""" @@ -71,22 +79,25 @@ class MainWindow(QMainWindow): def setup_initial_state(self): """Set up initial state""" - config = Config() - return { - "api_key": config.OPENAI_API_KEY or "", - "base_url": config.OPENAI_BASE_URL or "https://api.openai.com/v1", - "model": config.OPENAI_MODEL or "gpt-4o", - "theme": "Light", - "stop_hotkey": DEFAULT_STOP_HOTKEY, + # Get settings from settings manager + settings = self.settings_manager.get_settings() + + # Create state dictionary with settings and chat state + state = { + # Apply settings + **settings, + + # Chat state "messages": [], "chatbox_messages": [], "auth_validated": False, "responses": {}, "tools": {}, "tasks": [], - "only_n_most_recent_images": 2, "stop": False } + + return state def register_stop_hotkey(self): """Register the global stop hotkey""" @@ -184,28 +195,15 @@ class MainWindow(QMainWindow): # Main content area content_splitter = QSplitter(Qt.Orientation.Horizontal) - # Task list - task_widget = QWidget() - task_layout = QVBoxLayout(task_widget) - task_label = QLabel("Task List") - self.task_table = QTableWidget(0, 2) - self.task_table.setHorizontalHeaderLabels(["Status", "Task"]) - self.task_table.horizontalHeader().setSectionResizeMode(1, QHeaderView.ResizeMode.Stretch) - task_layout.addWidget(task_label) - task_layout.addWidget(self.task_table) + # Task panel + self.task_panel = TaskPanel() - # Chat area - chat_widget = QWidget() - chat_layout = QVBoxLayout(chat_widget) - chat_label = QLabel("Chat History") - self.chat_display = QTextEdit() - self.chat_display.setReadOnly(True) - chat_layout.addWidget(chat_label) - chat_layout.addWidget(self.chat_display) + # Chat panel + self.chat_panel = ChatPanel() # Add to splitter - content_splitter.addWidget(task_widget) - content_splitter.addWidget(chat_widget) + content_splitter.addWidget(self.task_panel) + content_splitter.addWidget(self.chat_panel) content_splitter.setSizes([int(self.width() * 0.2), int(self.width() * 0.8)]) # Add all components to main layout @@ -224,28 +222,24 @@ class MainWindow(QMainWindow): if result == QDialog.DialogCode.Accepted: # Get and apply new settings - settings = dialog.get_settings() + new_settings = dialog.get_settings() - # Check if stop hotkey changed - old_hotkey = self.state.get("stop_hotkey", DEFAULT_STOP_HOTKEY) - new_hotkey = settings["stop_hotkey"] + # Update settings in settings manager + changes = self.settings_manager.update_settings(new_settings) - self.state["model"] = settings["model"] - self.state["base_url"] = settings["base_url"] - self.state["api_key"] = settings["api_key"] - self.state["stop_hotkey"] = new_hotkey + # Update state with new settings + self.state.update(new_settings) - # Update theme if changed - if settings["theme"] != self.state.get("theme", "Light"): - self.state["theme"] = settings["theme"] + # Apply theme change if needed + if changes["theme_changed"]: self.apply_theme() - if settings["screen_region"]: - self.state["screen_region"] = settings["screen_region"] - # Update hotkey if changed - if old_hotkey != new_hotkey: + if changes["hotkey_changed"]: self.register_stop_hotkey() + + # Save settings to config + self.settings_manager.save_to_config() def process_input(self): """Process user input""" @@ -285,43 +279,10 @@ class MainWindow(QMainWindow): def update_ui(self, chatbox_messages, tasks): """Update UI display""" # Update chat display - self.chat_display.clear() - - for msg in chatbox_messages: - role = msg["role"] - content = msg["content"] - - # Set different formats based on role - format = QTextCharFormat() - if role == "user": - format.setForeground(QColor(0, 0, 255)) # Blue for user - self.chat_display.append("You:") - else: - format.setForeground(QColor(0, 128, 0)) # Green for AI - self.chat_display.append("AI:") - - # Add content - cursor = self.chat_display.textCursor() - cursor.movePosition(QTextCursor.MoveOperation.End) - - # Special handling for HTML content - if "<" in content and ">" in content: - self.chat_display.insertHtml(content) - self.chat_display.append("") # Add empty line - else: - self.chat_display.append(content) - self.chat_display.append("") # Add empty line - - # Scroll to bottom - self.chat_display.verticalScrollBar().setValue( - self.chat_display.verticalScrollBar().maximum() - ) + self.chat_panel.update_chat(chatbox_messages) # Update task table - self.task_table.setRowCount(len(tasks)) - for i, (status, task) in enumerate(tasks): - self.task_table.setItem(i, 0, QTableWidgetItem(status)) - self.task_table.setItem(i, 1, QTableWidgetItem(task)) + self.task_panel.update_tasks(tasks) def stop_process(self): """Stop processing - handles both button click and hotkey press""" @@ -331,9 +292,27 @@ class MainWindow(QMainWindow): if self.isMinimized(): self.showNormal() self.activateWindow() + self.chat_panel.append_message("⚠️ Stopped by user", "red") - self.chat_display.append("⚠️ Operation stopped by user") - self.register_stop_hotkey() + # Use non-modal dialog + learn_dialog = QMessageBox(self) + learn_dialog.setIcon(QMessageBox.Icon.Question) + learn_dialog.setWindowTitle("Learning Opportunity") + learn_dialog.setText("Would you like to show the correct steps to improve the system?") + learn_dialog.setStandardButtons(QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No) + learn_dialog.setDefaultButton(QMessageBox.StandardButton.No) + learn_dialog.setWindowModality(Qt.WindowModality.NonModal) + learn_dialog.show() + + # Connect signal to callback function + learn_dialog.buttonClicked.connect(self.handle_learn_dialog_response) + + def handle_learn_dialog_response(self, button): + if button.text() == "&Yes": + self.showMinimized() + self.recording_manager.start_demonstration() + # Update chat to show demonstration mode is active + self.chat_panel.append_message("📝 Demonstration mode activated. Please perform the correct actions.", "green") def clear_chat(self): """Clear chat history""" @@ -343,27 +322,21 @@ class MainWindow(QMainWindow): self.state["tools"] = {} self.state["tasks"] = [] - self.chat_display.clear() - self.task_table.setRowCount(0) + self.chat_panel.clear() + self.task_panel.clear() def closeEvent(self, event): - """Handle window close event""" - if hasattr(self, 'tray_icon') and self.tray_icon is not None and self.tray_icon.isVisible(): - self.hide() - event.ignore() - elif self.state.get("stop", False) and hasattr(self, 'worker') and self.worker is not None: - self.state["stop"] = False - event.ignore() - elif hasattr(self, 'worker') and self.worker is not None and self.worker.isRunning(): - reply = QMessageBox.question(self, 'Exit Confirmation', - '自动化任务仍在运行中，确定要退出程序吗？', - QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, - QMessageBox.StandardButton.No) - if reply == QMessageBox.StandardButton.Yes: - keyboard.unhook_all() - event.accept() - else: - event.ignore() - else: - keyboard.unhook_all() - event.accept() \ No newline at end of file + keyboard.unhook_all() + event.accept() + if hasattr(self, 'worker') and self.worker is not None: + self.worker.terminate() + +# 应用程序入口 +def main(): + app = QApplication(sys.argv) + window = MainWindow(sys.argv) + window.show() + sys.exit(app.exec()) # 注意PyQt6中不需要括号 + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/ui/recording_manager.py b/ui/recording_manager.py new file mode 100644 index 0000000..0c2c0b0 --- /dev/null +++ b/ui/recording_manager.py @@ -0,0 +1,81 @@ +""" +Recording manager for autoMate +Handles recording and demonstration functionality +""" +import util.auto_control as auto_control +from ui.recording_panel import RecordingIndicator +from ui.demonstration_panel import DemonstrationPanel + +class RecordingManager: + def __init__(self, parent=None): + self.parent = parent + self.recording_in_progress = False + self.recording_indicator = None + self.demo_panel = None + self.demonstration_mode = False + + def start_recording(self): + """Start recording user actions""" + if not self.recording_in_progress: + self.recording_in_progress = True + + # 最小化主窗口 + if self.parent: + self.parent.showMinimized() + + # 显示录制指示器 + self.recording_indicator = RecordingIndicator(stop_callback=self.stop_recording) + self.recording_indicator.show() + + # 开始监听用户动作 + auto_control.start_monitoring() + + def stop_recording(self): + """Stop recording user actions""" + if self.recording_in_progress: + self.recording_in_progress = False + + # 停止监听用户动作 + auto_control.stop_monitoring() + + # 关闭录制指示器 + if self.recording_indicator: + self.recording_indicator.close() + self.recording_indicator = None + + # 恢复主窗口 + if self.parent: + self.parent.showNormal() + + def start_demonstration(self): + """Start demonstration mode for system learning""" + # Set demonstration mode flag + self.demonstration_mode = True + + # 隐藏主窗口 + if self.parent: + self.parent.showMinimized() + + # 创建并显示独立的演示控制面板 + self.demo_panel = DemonstrationPanel(stop_callback=self.stop_demonstration) + self.demo_panel.show() + + # 开始监听用户动作 + auto_control.start_monitoring() + + def stop_demonstration(self): + """Stop demonstration mode and process the recorded actions""" + # 停止监听用户动作 + auto_control.stop_monitoring() + + # 关闭独立的演示控制面板 + if self.demo_panel: + self.demo_panel.close() + self.demo_panel = None + + # 恢复主窗口 + if self.parent: + self.parent.showNormal() + + # Reset state + self.demonstration_mode = False \ No newline at end of file diff --git a/ui/recording_panel.py b/ui/recording_panel.py new file mode 100644 index 0000000..e36d4c8 --- /dev/null +++ b/ui/recording_panel.py @@ -0,0 +1,43 @@ +""" +Recording indicator panel for autoMate +""" +from PyQt6.QtWidgets import QWidget, QVBoxLayout, QHBoxLayout, QLabel, QPushButton, QApplication +from PyQt6.QtCore import Qt, QPoint + +class RecordingIndicator(QWidget): + def __init__(self, parent=None, stop_callback=None): + super().__init__(parent, Qt.WindowType.WindowStaysOnTopHint | Qt.WindowType.FramelessWindowHint) + self.stop_callback = stop_callback + self.setup_ui() + self.position_to_bottom_right() + + def setup_ui(self): + layout = QVBoxLayout() + + # Recording status label + self.status_label = QLabel("Recording in progress") + self.status_label.setStyleSheet("color: red; font-weight: bold;") + layout.addWidget(self.status_label) + + # Stop button + self.stop_button = QPushButton("Stop Recording") + self.stop_button.clicked.connect(self.on_stop_clicked) + layout.addWidget(self.stop_button) + + self.setLayout(layout) + self.resize(200, 100) + self.setStyleSheet("background-color: #f0f0f0; border: 1px solid #999;") + + def position_to_bottom_right(self): + screen = QApplication.primaryScreen() + screen_geometry = screen.availableGeometry() + window_geometry = self.frameGeometry() + position = QPoint( + screen_geometry.width() - window_geometry.width() - 20, + screen_geometry.height() - window_geometry.height() - 20 + ) + self.move(position) + + def on_stop_clicked(self): + if self.stop_callback: + self.stop_callback() \ No newline at end of file diff --git a/ui/settings_manager.py b/ui/settings_manager.py new file mode 100644 index 0000000..83bb512 --- /dev/null +++ b/ui/settings_manager.py @@ -0,0 +1,59 @@ +""" +Settings manager for autoMate +Handles loading, saving, and updating application settings +""" +from xbrain.utils.config import Config +from ui.hotkey_edit import DEFAULT_STOP_HOTKEY + +class SettingsManager: + """Manages application settings""" + + def __init__(self): + self.config = Config() + self.settings = self.load_initial_settings() + + def load_initial_settings(self): + """Load initial settings from config""" + return { + "api_key": self.config.OPENAI_API_KEY or "", + "base_url": self.config.OPENAI_BASE_URL or "https://api.openai.com/v1", + "model": self.config.OPENAI_MODEL or "gpt-4o", + "theme": "Light", + "stop_hotkey": DEFAULT_STOP_HOTKEY, + "only_n_most_recent_images": 2, + "screen_region": None + } + + def get_settings(self): + """Get current settings""" + return self.settings + + def update_settings(self, new_settings): + """Update settings""" + # Track if hotkey changed + hotkey_changed = False + if "stop_hotkey" in new_settings and new_settings["stop_hotkey"] != self.settings.get("stop_hotkey"): + hotkey_changed = True + + # Track if theme changed + theme_changed = False + if "theme" in new_settings and new_settings["theme"] != self.settings.get("theme"): + theme_changed = True + + # Update settings + self.settings.update(new_settings) + + return { + "hotkey_changed": hotkey_changed, + "theme_changed": theme_changed + } + + def save_to_config(self): + """Save settings to config file""" + # Update config with current settings + self.config.OPENAI_API_KEY = self.settings.get("api_key", "") + self.config.OPENAI_BASE_URL = self.settings.get("base_url", "https://api.openai.com/v1") + self.config.OPENAI_MODEL = self.settings.get("model", "gpt-4o") + + # Save config to file + self.config.save() \ No newline at end of file diff --git a/ui/task_panel.py b/ui/task_panel.py new file mode 100644 index 0000000..9e36695 --- /dev/null +++ b/ui/task_panel.py @@ -0,0 +1,30 @@ +""" +Task panel for autoMate +""" +from PyQt6.QtWidgets import QWidget, QVBoxLayout, QLabel, QTableWidget, QTableWidgetItem, QHeaderView + +class TaskPanel(QWidget): + def __init__(self, parent=None): + super().__init__(parent) + self.setup_ui() + + def setup_ui(self): + """Initialize task panel UI""" + task_layout = QVBoxLayout(self) + task_label = QLabel("Task List") + self.task_table = QTableWidget(0, 2) + self.task_table.setHorizontalHeaderLabels(["Status", "Task"]) + self.task_table.horizontalHeader().setSectionResizeMode(1, QHeaderView.ResizeMode.Stretch) + task_layout.addWidget(task_label) + task_layout.addWidget(self.task_table) + + def update_tasks(self, tasks): + """Update task table with new tasks""" + self.task_table.setRowCount(len(tasks)) + for i, (status, task) in enumerate(tasks): + self.task_table.setItem(i, 0, QTableWidgetItem(status)) + self.task_table.setItem(i, 1, QTableWidgetItem(task)) + + def clear(self): + """Clear all tasks""" + self.task_table.setRowCount(0) \ No newline at end of file diff --git a/util/auto_control.py b/util/auto_control.py index b2852d9..aec465d 100644 --- a/util/auto_control.py +++ b/util/auto_control.py @@ -5,7 +5,7 @@ import time # Add the project root directory to Python path sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from auto_control.agent.vision_agent import VisionAgent -from util.download_weights import MODEL_DIR +from util.download_weights import OMNI_PARSER_DIR from pynput import mouse, keyboard # Now you can import from auto_control @@ -81,7 +81,7 @@ class AutoControl: if key == keyboard.Key.esc: print("self.auto_list", self.auto_list) - vision_agent = VisionAgent(yolo_model_path=os.path.join(MODEL_DIR, "icon_detect", "model.pt")) + vision_agent = VisionAgent(yolo_model_path=os.path.join(OMNI_PARSER_DIR, "icon_detect", "model.pt")) for item in self.auto_list: element_list =vision_agent(str(item["path"])) @@ -119,6 +119,25 @@ class AutoControl: return False +# User action monitoring module + +def start_monitoring(): + """ + Start monitoring user actions (keyboard and mouse) + """ + print("Started monitoring user actions") + # Implementation for monitoring user actions + # This could use libraries like pynput, pyautogui, etc. + +def stop_monitoring(): + """ + Stop monitoring user actions + """ + print("Stopped monitoring user actions") + # Implementation to stop monitoring + +# Additional functionality for processing recorded actions + if __name__ == "__main__": auto_control = AutoControl() auto_control.start_listen()