diff --git a/.gitignore b/.gitignore index 4d9ae04..d550f45 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .env llm_debug.log -__pycache__/ \ No newline at end of file +__pycache__/ +debug_screenshots/ \ No newline at end of file diff --git a/ClaudeCode.md b/ClaudeCode.md index 00322f3..42bb7a4 100644 --- a/ClaudeCode.md +++ b/ClaudeCode.md @@ -75,10 +75,10 @@ Wolf Chat 是一個基於 MCP (Modular Capability Provider) 框架的聊天機 系統使用基於圖像辨識的方法監控遊戲聊天界面： -1. **泡泡檢測**：通過辨識聊天泡泡的角落圖案定位聊天訊息，區分一般用戶與機器人 +1. **泡泡檢測**：通過辨識聊天泡泡的左上角 (TL) 和右下角 (BR) 角落圖案定位聊天訊息。系統能區分一般用戶泡泡和機器人泡泡。**為了適應玩家可能使用的不同聊天泡泡外觀 (skin)，一般用戶泡泡的偵測機制已被擴充，可以同時尋找多組不同的角落模板 (例如 `corner_tl_type2.png`, `corner_br_type2.png` 等)，提高了對自訂外觀的兼容性。機器人泡泡目前僅偵測預設的角落模板。** 2. **關鍵字檢測**：在泡泡區域內搜尋 "wolf" 或 "Wolf" 關鍵字圖像 3. **內容獲取**：點擊關鍵字位置，使用剪貼板複製聊天內容 -4. **發送者識別**：通過點擊頭像，導航菜單，複製用戶名稱 +4. **發送者識別**：**關鍵步驟** - 系統會根據**偵測到關鍵字的那個特定聊天泡泡**的左上角座標，計算出頭像的點擊位置（目前水平偏移量為 -55 像素）。這確保了點擊的是觸發訊息的發送者頭像，而不是其他位置的頭像。接著通過點擊計算出的頭像位置，導航菜單，最終複製用戶名稱。 5. **防重複處理**：使用位置比較和內容歷史記錄防止重複回應 #### LLM 整合 @@ -167,6 +167,31 @@ Wolf Chat 是一個基於 MCP (Modular Capability Provider) 框架的聊天機 這些優化確保了即使在複雜工具調用後，Wolfhart 也能保持角色一致性，並提供合適的回應。無效回應不再發送到遊戲，提高了用戶體驗。 +## 最近改進（2025-04-18） + +### 支援多種一般聊天泡泡外觀，並修正先前錯誤配置 + +- **UI 互動模塊 (`ui_interaction.py`)**： + - **修正**：先前錯誤地將多外觀支援應用於機器人泡泡。現已修正 `find_dialogue_bubbles` 函數，使其能夠載入並搜尋多組**一般用戶**泡泡的角落模板（例如 `corner_tl_type2.png`, `corner_br_type2.png` 等）。 + - 允許任何類型的一般用戶左上角與任何類型的一般用戶右下角進行配對，只要符合幾何條件。 + - 機器人泡泡的偵測恢復為僅使用預設的 `bot_corner_tl.png` 和 `bot_corner_br.png` 模板。 + - 這提高了對使用了自訂聊天泡泡外觀的**一般玩家**訊息的偵測能力。 +- **模板文件**： + - 在 `ui_interaction.py` 中為一般角落定義了新類型模板的路徑（`_type2`, `_type3`）。 + - **注意：** 需要在 `templates` 資料夾中實際添加對應的 `corner_tl_type2.png`, `corner_br_type2.png` 等圖片檔案才能生效。 +- **文件更新 (`ClaudeCode.md`)**： + - 在「技術實現」部分更新了泡泡檢測的說明。 + - 添加了此「最近改進」條目，並修正了先前的描述。 + +### 頭像點擊偏移量調整 + +- **UI 互動模塊 (`ui_interaction.py`)**： + - 將 `AVATAR_OFFSET_X` 常數的值從 `-50` 調整為 `-55`。 + - 這統一了常規關鍵字觸發流程和 `remove_user_position` 功能中計算頭像點擊位置時使用的水平偏移量。 +- **文件更新 (`ClaudeCode.md`)**： + - 在「技術實現」的「發送者識別」部分強調了點擊位置是相對於觸發泡泡計算的，並註明了新的偏移量。 + - 添加了此「最近改進」條目。 + ## 開發建議 ### 優化方向 diff --git a/config.py b/config.py index 13e410f..4242b66 100644 --- a/config.py +++ b/config.py @@ -27,24 +27,24 @@ exa_config_dict = {"exaApiKey": EXA_API_KEY if EXA_API_KEY else "YOUR_EXA_KEY_MI # For cmd /c on Windows, embedding escaped JSON often works like this: exa_config_arg_string = json.dumps(json.dumps(exa_config_dict)) # Double dump for cmd escaping? Or just one? Test needed. # Let's try single dump first, often sufficient if passed correctly by subprocess -exa_config_arg_string_single_dump = json.dumps(exa_config_dict) +exa_config_arg_string_single_dump = json.dumps(exa_config_dict) # Use this one # --- MCP Server Configuration --- MCP_SERVERS = { - "exa": { - "command": "cmd", - "args": [ - "/c", - "npx", - "-y", - "@smithery/cli@latest", - "run", - "exa", - "--config", - # Pass the dynamically created config string with the environment variable key - exa_config_arg_string # Use the properly escaped variable - ], - }, + # "exa": { # Temporarily commented out to prevent blocking startup + # "command": "cmd", + # "args": [ + # "/c", + # "npx", + # "-y", + # "@smithery/cli@latest", + # "run", + # "exa", + # "--config", + # # Pass the dynamically created config string with the environment variable key + # exa_config_arg_string_single_dump # Use the single dump variable + # ], + # }, "servers": { "command": "npx", "args": [ @@ -71,5 +71,5 @@ WINDOW_TITLE = "Last War-Survival Game" # --- Print loaded keys for verification (Optional - BE CAREFUL!) --- # print(f"DEBUG: Loaded OPENAI_API_KEY: {'*' * (len(OPENAI_API_KEY) - 4) + OPENAI_API_KEY[-4:] if OPENAI_API_KEY else 'Not Found'}") -# print(f"DEBUG: Loaded EXA_API_KEY: {'*' * (len(EXA_API_KEY) - 4) + EXA_API_KEY[-4:] if EXA_API_KEY else 'Not Found'}") -# print(f"DEBUG: Exa args: {MCP_SERVERS['exa']['args']}") \ No newline at end of file +print(f"DEBUG: Loaded EXA_API_KEY: {'*' * (len(EXA_API_KEY) - 4) + EXA_API_KEY[-4:] if EXA_API_KEY else 'Not Found'}") # Uncommented Exa key check +# print(f"DEBUG: Exa args: {MCP_SERVERS['exa']['args']}") diff --git a/llm_interaction.py b/llm_interaction.py index d147720..7e18ff3 100644 --- a/llm_interaction.py +++ b/llm_interaction.py @@ -165,10 +165,15 @@ You MUST respond in the following JSON format: Parameters: `names` (array of strings) Usage: Access specific entities you know exist in the graph. + **Game Actions:** + - `remove_position`: Initiate the process to remove a user's assigned position/role. + Parameters: (none) - The context (triggering message) is handled separately. + Usage: Use ONLY when the user explicitly requests a position removal AND you, as Wolfhart, decide to grant the request based on the interaction's tone, politeness, and perceived intent (e.g., not malicious or a prank). Your decision should reflect Wolfhart's personality (calm, strategic, potentially dismissive of rudeness or foolishness). If you decide to remove the position, include this command alongside your dialogue response. + 3. `thoughts` (OPTIONAL): Your internal analysis that won't be shown to users. Use this for your reasoning process. - - Think about whether you need to use memory tools or web search - - Analyze the user's question and determine what information is needed - - Plan your approach before responding + - Think about whether you need to use memory tools or web search. + - Analyze the user's message: Is it a request to remove a position? If so, evaluate its politeness and intent from Wolfhart's perspective. Decide whether to issue the `remove_position` command. + - Plan your approach before responding. **VERY IMPORTANT Instructions:** @@ -691,4 +696,3 @@ async def _execute_single_tool_call(tool_call, mcp_sessions, available_mcp_tools f"Tool: {function_name}\nFormatted Response: {json.dumps(response, ensure_ascii=False, indent=2)}") return response - diff --git a/main.py b/main.py index b7cbc1d..395b8f7 100644 --- a/main.py +++ b/main.py @@ -227,12 +227,15 @@ async def run_main_with_exit_stack(): sender_name = trigger_data.get('sender') bubble_text = trigger_data.get('text') + bubble_region = trigger_data.get('bubble_region') # <-- Extract bubble_region print(f"\n--- Received trigger from UI ---") print(f" Sender: {sender_name}") print(f" Content: {bubble_text[:100]}...") + if bubble_region: + print(f" Bubble Region: {bubble_region}") # <-- Log bubble_region - if not sender_name or not bubble_text: - print("Warning: Received incomplete trigger data, skipping.") + if not sender_name or not bubble_text: # bubble_region is optional context, don't fail if missing + print("Warning: Received incomplete trigger data (missing sender or text), skipping.") # No task_done needed for standard queue continue @@ -257,11 +260,31 @@ async def run_main_with_exit_stack(): print(f"Processing {len(commands)} command(s)...") for cmd in commands: cmd_type = cmd.get("type", "") - cmd_params = cmd.get("parameters", {}) - # 預留位置：在這裡添加命令處理邏輯 - print(f"Command type: {cmd_type}, parameters: {cmd_params}") - # TODO: 實現各類命令的處理邏輯 - + cmd_params = cmd.get("parameters", {}) # Parameters might be empty for remove_position + + # --- Command Processing --- + if cmd_type == "remove_position": + if bubble_region: # Check if we have the context + print("Sending 'remove_position' command to UI thread...") + command_to_send = { + 'action': 'remove_position', + 'trigger_bubble_region': bubble_region # Pass the region + } + try: + await loop.run_in_executor(None, command_queue.put, command_to_send) + print("Command placed in queue.") + except Exception as q_err: + print(f"Error putting remove_position command in queue: {q_err}") + else: + print("Error: Cannot process 'remove_position' command without bubble_region context.") + # Add other command handling here if needed + # elif cmd_type == "some_other_command": + # # Handle other commands + # pass + else: + print(f"Received unhandled command type: {cmd_type}, parameters: {cmd_params}") + # --- End Command Processing --- + # 記錄思考過程 (如果有的話) thoughts = bot_response_data.get("thoughts", "") if thoughts: diff --git a/templates/base.png b/templates/base.png new file mode 100644 index 0000000..7fb93fa Binary files /dev/null and b/templates/base.png differ diff --git a/templates/capitol/black_arrow_down.png b/templates/capitol/black_arrow_down.png new file mode 100644 index 0000000..824abd1 Binary files /dev/null and b/templates/capitol/black_arrow_down.png differ diff --git a/templates/capitol/capitol_#11.png b/templates/capitol/capitol_#11.png new file mode 100644 index 0000000..1b2c193 Binary files /dev/null and b/templates/capitol/capitol_#11.png differ diff --git a/templates/capitol/close_button.png b/templates/capitol/close_button.png new file mode 100644 index 0000000..38fcdd8 Binary files /dev/null and b/templates/capitol/close_button.png differ diff --git a/templates/capitol/confirm.png b/templates/capitol/confirm.png new file mode 100644 index 0000000..dc2556a Binary files /dev/null and b/templates/capitol/confirm.png differ diff --git a/templates/capitol/dismiss.png b/templates/capitol/dismiss.png new file mode 100644 index 0000000..eb8e87a Binary files /dev/null and b/templates/capitol/dismiss.png differ diff --git a/templates/capitol/page_DEVELOPMENT.png b/templates/capitol/page_DEVELOPMENT.png new file mode 100644 index 0000000..927db88 Binary files /dev/null and b/templates/capitol/page_DEVELOPMENT.png differ diff --git a/templates/capitol/page_INTERIOR.png b/templates/capitol/page_INTERIOR.png new file mode 100644 index 0000000..1394345 Binary files /dev/null and b/templates/capitol/page_INTERIOR.png differ diff --git a/templates/capitol/page_SCIENCE.png b/templates/capitol/page_SCIENCE.png new file mode 100644 index 0000000..5daf207 Binary files /dev/null and b/templates/capitol/page_SCIENCE.png differ diff --git a/templates/capitol/page_SECURITY.png b/templates/capitol/page_SECURITY.png new file mode 100644 index 0000000..6617409 Binary files /dev/null and b/templates/capitol/page_SECURITY.png differ diff --git a/templates/capitol/page_STRATEGY.png b/templates/capitol/page_STRATEGY.png new file mode 100644 index 0000000..3e43e4e Binary files /dev/null and b/templates/capitol/page_STRATEGY.png differ diff --git a/templates/capitol/position_development.png b/templates/capitol/position_development.png new file mode 100644 index 0000000..ffff6a2 Binary files /dev/null and b/templates/capitol/position_development.png differ diff --git a/templates/capitol/position_interior.png b/templates/capitol/position_interior.png new file mode 100644 index 0000000..7be95d8 Binary files /dev/null and b/templates/capitol/position_interior.png differ diff --git a/templates/capitol/position_science.png b/templates/capitol/position_science.png new file mode 100644 index 0000000..e6d354d Binary files /dev/null and b/templates/capitol/position_science.png differ diff --git a/templates/capitol/position_security.png b/templates/capitol/position_security.png new file mode 100644 index 0000000..17a9e4e Binary files /dev/null and b/templates/capitol/position_security.png differ diff --git a/templates/capitol/position_strategy.png b/templates/capitol/position_strategy.png new file mode 100644 index 0000000..3c1e2a7 Binary files /dev/null and b/templates/capitol/position_strategy.png differ diff --git a/templates/capitol/president_title.png b/templates/capitol/president_title.png new file mode 100644 index 0000000..41a8e04 Binary files /dev/null and b/templates/capitol/president_title.png differ diff --git a/templates/capitol/president_title1.png b/templates/capitol/president_title1.png new file mode 100644 index 0000000..5f626b4 Binary files /dev/null and b/templates/capitol/president_title1.png differ diff --git a/templates/capitol/president_title2.png b/templates/capitol/president_title2.png new file mode 100644 index 0000000..c404487 Binary files /dev/null and b/templates/capitol/president_title2.png differ diff --git a/templates/corner_br_type2.png b/templates/corner_br_type2.png new file mode 100644 index 0000000..a526410 Binary files /dev/null and b/templates/corner_br_type2.png differ diff --git a/templates/corner_br_type3.png b/templates/corner_br_type3.png new file mode 100644 index 0000000..a1ed184 Binary files /dev/null and b/templates/corner_br_type3.png differ diff --git a/templates/corner_tl.png b/templates/corner_tl.png index 9fe6a2a..8bdd5b0 100644 Binary files a/templates/corner_tl.png and b/templates/corner_tl.png differ diff --git a/templates/corner_tl_type2.png b/templates/corner_tl_type2.png new file mode 100644 index 0000000..1e925a7 Binary files /dev/null and b/templates/corner_tl_type2.png differ diff --git a/templates/corner_tl_type3.png b/templates/corner_tl_type3.png new file mode 100644 index 0000000..a0ef7d9 Binary files /dev/null and b/templates/corner_tl_type3.png differ diff --git a/templates/keyword_wolf_lower_type3.png b/templates/keyword_wolf_lower_type3.png new file mode 100644 index 0000000..b31dcce Binary files /dev/null and b/templates/keyword_wolf_lower_type3.png differ diff --git a/templates/keyword_wolf_upper_type3.png b/templates/keyword_wolf_upper_type3.png new file mode 100644 index 0000000..cd96185 Binary files /dev/null and b/templates/keyword_wolf_upper_type3.png differ diff --git a/templates/positions/development.png b/templates/positions/development.png new file mode 100644 index 0000000..26ae2cd Binary files /dev/null and b/templates/positions/development.png differ diff --git a/templates/positions/interior.png b/templates/positions/interior.png new file mode 100644 index 0000000..f5b76c4 Binary files /dev/null and b/templates/positions/interior.png differ diff --git a/templates/positions/science.png b/templates/positions/science.png new file mode 100644 index 0000000..b816600 Binary files /dev/null and b/templates/positions/science.png differ diff --git a/templates/positions/security.png b/templates/positions/security.png new file mode 100644 index 0000000..c7db898 Binary files /dev/null and b/templates/positions/security.png differ diff --git a/templates/positions/strategy.png b/templates/positions/strategy.png new file mode 100644 index 0000000..e53efe1 Binary files /dev/null and b/templates/positions/strategy.png differ diff --git a/ui_interaction.py b/ui_interaction.py index 045cfdf..bbd9db7 100644 --- a/ui_interaction.py +++ b/ui_interaction.py @@ -19,19 +19,41 @@ SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) TEMPLATE_DIR = os.path.join(SCRIPT_DIR, "templates") os.makedirs(TEMPLATE_DIR, exist_ok=True) +# --- Debugging --- +DEBUG_SCREENSHOT_DIR = os.path.join(SCRIPT_DIR, "debug_screenshots") +MAX_DEBUG_SCREENSHOTS = 5 +os.makedirs(DEBUG_SCREENSHOT_DIR, exist_ok=True) +# --- End Debugging --- + # --- Template Paths (Consider moving to config.py or loading dynamically) --- # Bubble Corners CORNER_TL_IMG = os.path.join(TEMPLATE_DIR, "corner_tl.png") -CORNER_TR_IMG = os.path.join(TEMPLATE_DIR, "corner_tr.png") -CORNER_BL_IMG = os.path.join(TEMPLATE_DIR, "corner_bl.png") +# CORNER_TR_IMG = os.path.join(TEMPLATE_DIR, "corner_tr.png") # Unused +# CORNER_BL_IMG = os.path.join(TEMPLATE_DIR, "corner_bl.png") # Unused CORNER_BR_IMG = os.path.join(TEMPLATE_DIR, "corner_br.png") +# --- Additional Regular Bubble Types (Skins) --- +CORNER_TL_TYPE2_IMG = os.path.join(TEMPLATE_DIR, "corner_tl_type2.png") # Added +CORNER_BR_TYPE2_IMG = os.path.join(TEMPLATE_DIR, "corner_br_type2.png") # Added +CORNER_TL_TYPE3_IMG = os.path.join(TEMPLATE_DIR, "corner_tl_type3.png") # Added +CORNER_BR_TYPE3_IMG = os.path.join(TEMPLATE_DIR, "corner_br_type3.png") # Added +# --- End Additional Regular Types --- BOT_CORNER_TL_IMG = os.path.join(TEMPLATE_DIR, "bot_corner_tl.png") -BOT_CORNER_TR_IMG = os.path.join(TEMPLATE_DIR, "bot_corner_tr.png") -BOT_CORNER_BL_IMG = os.path.join(TEMPLATE_DIR, "bot_corner_bl.png") +# BOT_CORNER_TR_IMG = os.path.join(TEMPLATE_DIR, "bot_corner_tr.png") # Unused +# BOT_CORNER_BL_IMG = os.path.join(TEMPLATE_DIR, "bot_corner_bl.png") # Unused BOT_CORNER_BR_IMG = os.path.join(TEMPLATE_DIR, "bot_corner_br.png") +# --- Additional Bot Bubble Types (Skins) --- +# Type 2 +BOT_CORNER_TL_TYPE2_IMG = os.path.join(TEMPLATE_DIR, "bot_corner_tl_type2.png") +BOT_CORNER_BR_TYPE2_IMG = os.path.join(TEMPLATE_DIR, "bot_corner_br_type2.png") +# Type 3 +BOT_CORNER_TL_TYPE3_IMG = os.path.join(TEMPLATE_DIR, "bot_corner_tl_type3.png") +BOT_CORNER_BR_TYPE3_IMG = os.path.join(TEMPLATE_DIR, "bot_corner_br_type3.png") +# --- End Additional Types --- # Keywords KEYWORD_wolf_LOWER_IMG = os.path.join(TEMPLATE_DIR, "keyword_wolf_lower.png") KEYWORD_Wolf_UPPER_IMG = os.path.join(TEMPLATE_DIR, "keyword_wolf_upper.png") +KEYWORD_wolf_LOWER_TYPE3_IMG = os.path.join(TEMPLATE_DIR, "keyword_wolf_lower_type3.png") # Added for type3 bubbles +KEYWORD_Wolf_UPPER_TYPE3_IMG = os.path.join(TEMPLATE_DIR, "keyword_wolf_upper_type3.png") # Added for type3 bubbles # UI Elements COPY_MENU_ITEM_IMG = os.path.join(TEMPLATE_DIR, "copy_menu_item.png") PROFILE_OPTION_IMG = os.path.join(TEMPLATE_DIR, "profile_option.png") @@ -42,10 +64,38 @@ CHAT_INPUT_IMG = os.path.join(TEMPLATE_DIR, "chat_input.png") PROFILE_NAME_PAGE_IMG = os.path.join(TEMPLATE_DIR, "Profile_Name_page.png") PROFILE_PAGE_IMG = os.path.join(TEMPLATE_DIR, "Profile_page.png") CHAT_ROOM_IMG = os.path.join(TEMPLATE_DIR, "chat_room.png") +BASE_SCREEN_IMG = os.path.join(TEMPLATE_DIR, "base.png") # Added for navigation +WORLD_MAP_IMG = os.path.join(TEMPLATE_DIR, "World_map.png") # Added for navigation # Add World/Private chat identifiers later WORLD_CHAT_IMG = os.path.join(TEMPLATE_DIR, "World_Label_normal.png") # Example PRIVATE_CHAT_IMG = os.path.join(TEMPLATE_DIR, "Private_Label_normal.png") # Example +# Position Icons (Near Bubble) +POS_DEV_IMG = os.path.join(TEMPLATE_DIR, "positions", "development.png") +POS_INT_IMG = os.path.join(TEMPLATE_DIR, "positions", "interior.png") +POS_SCI_IMG = os.path.join(TEMPLATE_DIR, "positions", "science.png") +POS_SEC_IMG = os.path.join(TEMPLATE_DIR, "positions", "security.png") +POS_STR_IMG = os.path.join(TEMPLATE_DIR, "positions", "strategy.png") + +# Capitol Page Elements +CAPITOL_BUTTON_IMG = os.path.join(TEMPLATE_DIR, "capitol", "capitol_#11.png") +PRESIDENT_TITLE_IMG = os.path.join(TEMPLATE_DIR, "capitol", "president_title.png") +POS_BTN_DEV_IMG = os.path.join(TEMPLATE_DIR, "capitol", "position_development.png") +POS_BTN_INT_IMG = os.path.join(TEMPLATE_DIR, "capitol", "position_interior.png") +POS_BTN_SCI_IMG = os.path.join(TEMPLATE_DIR, "capitol", "position_science.png") +POS_BTN_SEC_IMG = os.path.join(TEMPLATE_DIR, "capitol", "position_security.png") +POS_BTN_STR_IMG = os.path.join(TEMPLATE_DIR, "capitol", "position_strategy.png") +PAGE_DEV_IMG = os.path.join(TEMPLATE_DIR, "capitol", "page_DEVELOPMENT.png") +PAGE_INT_IMG = os.path.join(TEMPLATE_DIR, "capitol", "page_INTERIOR.png") +PAGE_SCI_IMG = os.path.join(TEMPLATE_DIR, "capitol", "page_SCIENCE.png") +PAGE_SEC_IMG = os.path.join(TEMPLATE_DIR, "capitol", "page_SECURITY.png") +PAGE_STR_IMG = os.path.join(TEMPLATE_DIR, "capitol", "page_STRATEGY.png") +DISMISS_BUTTON_IMG = os.path.join(TEMPLATE_DIR, "capitol", "dismiss.png") +CONFIRM_BUTTON_IMG = os.path.join(TEMPLATE_DIR, "capitol", "confirm.png") +CLOSE_BUTTON_IMG = os.path.join(TEMPLATE_DIR, "capitol", "close_button.png") +BACK_ARROW_IMG = os.path.join(TEMPLATE_DIR, "capitol", "black_arrow_down.png") + + # --- Operation Parameters (Consider moving to config.py) --- CHAT_INPUT_REGION = None # Example: (100, 800, 500, 50) CHAT_INPUT_CENTER_X = 400 @@ -53,7 +103,7 @@ CHAT_INPUT_CENTER_Y = 1280 SCREENSHOT_REGION = None CONFIDENCE_THRESHOLD = 0.8 STATE_CONFIDENCE_THRESHOLD = 0.7 -AVATAR_OFFSET_X = -50 +AVATAR_OFFSET_X = -55 # Adjusted as per user request (was -50) BBOX_SIMILARITY_TOLERANCE = 10 RECENT_TEXT_HISTORY_MAXLEN = 5 # This state likely belongs in the coordinator @@ -64,6 +114,7 @@ def are_bboxes_similar(bbox1: Optional[Tuple[int, int, int, int]], """Check if two bounding boxes' top-left corners are close.""" if bbox1 is None or bbox2 is None: return False + # Compare based on bbox top-left (index 0 and 1) return abs(bbox1[0] - bbox2[0]) <= tolerance and abs(bbox1[1] - bbox2[1]) <= tolerance # ============================================================================== @@ -81,7 +132,7 @@ class DetectionModule: print("DetectionModule initialized.") def _find_template(self, template_key: str, confidence: Optional[float] = None, region: Optional[Tuple[int, int, int, int]] = None, grayscale: bool = False) -> List[Tuple[int, int]]: - """Internal helper to find a template by its key.""" + """Internal helper to find a template by its key. Returns list of CENTER coordinates.""" template_path = self.templates.get(template_key) if not template_path: print(f"Error: Template key '{template_key}' not found in provided templates.") @@ -99,9 +150,11 @@ class DetectionModule: current_confidence = confidence if confidence is not None else self.confidence try: + # locateAllOnScreen returns Box objects (left, top, width, height) matches = pyautogui.locateAllOnScreen(template_path, region=current_region, confidence=current_confidence, grayscale=grayscale) if matches: for box in matches: + # Calculate center coordinates from the Box object center_x = box.left + box.width // 2 center_y = box.top + box.height // 2 locations.append((center_x, center_y)) @@ -111,88 +164,172 @@ class DetectionModule: print(f"Error finding template '{template_key}' ({template_path}): {e}") return [] + def _find_template_raw(self, template_key: str, confidence: Optional[float] = None, region: Optional[Tuple[int, int, int, int]] = None, grayscale: bool = False) -> List[Tuple[int, int, int, int]]: + """Internal helper to find a template by its key. Returns list of raw Box tuples (left, top, width, height).""" + template_path = self.templates.get(template_key) + if not template_path: + print(f"Error: Template key '{template_key}' not found in provided templates.") + return [] + if not os.path.exists(template_path): + if template_path not in self._warned_paths: + print(f"Error: Template image doesn't exist: {template_path}") + self._warned_paths.add(template_path) + return [] + + locations = [] + current_region = region if region is not None else self.region + current_confidence = confidence if confidence is not None else self.confidence + try: + # --- Temporary Debug Print --- + print(f"DEBUG: Searching for template '{template_key}' with confidence {current_confidence}...") + # --- End Temporary Debug Print --- + matches = pyautogui.locateAllOnScreen(template_path, region=current_region, confidence=current_confidence, grayscale=grayscale) + match_count = 0 # Initialize count + if matches: + for box in matches: + locations.append((box.left, box.top, box.width, box.height)) + match_count += 1 # Increment count + # --- Temporary Debug Print --- + print(f"DEBUG: Found {match_count} instance(s) of template '{template_key}'.") + # --- End Temporary Debug Print --- + return locations + except Exception as e: + print(f"Error finding template raw '{template_key}' ({template_path}): {e}") + return [] + def find_elements(self, template_keys: List[str], confidence: Optional[float] = None, region: Optional[Tuple[int, int, int, int]] = None) -> Dict[str, List[Tuple[int, int]]]: - """Find multiple templates by their keys.""" + """Find multiple templates by their keys. Returns center coordinates.""" results = {} for key in template_keys: results[key] = self._find_template(key, confidence=confidence, region=region) return results - def find_dialogue_bubbles(self) -> List[Tuple[Tuple[int, int, int, int], bool]]: + def find_dialogue_bubbles(self) -> List[Dict[str, Any]]: """ - Scan screen for regular and bot bubble corners and pair them. - Returns list of (bbox, is_bot_flag). Basic matching logic. + Scan screen for regular and multiple types of bot bubble corners and pair them. + Returns a list of dictionaries, each containing: + {'bbox': (tl_x, tl_y, br_x, br_y), 'is_bot': bool, 'tl_coords': (original_tl_x, original_tl_y)} """ - all_bubbles_with_type = [] + all_bubbles_info = [] + processed_tls = set() # Keep track of TL corners already used in a bubble - # Find corners using the internal helper - tl_corners = self._find_template('corner_tl') - br_corners = self._find_template('corner_br') - bot_tl_corners = self._find_template('bot_corner_tl') - bot_br_corners = self._find_template('bot_corner_br') + # --- Find ALL Regular Bubble Corners (Raw Coordinates) --- + regular_tl_keys = ['corner_tl', 'corner_tl_type2', 'corner_tl_type3'] # Modified + regular_br_keys = ['corner_br', 'corner_br_type2', 'corner_br_type3'] # Modified - # Match regular bubbles - processed_tls = set() - if tl_corners and br_corners: - for i, tl in enumerate(tl_corners): - if i in processed_tls: continue - potential_br = None + all_regular_tl_boxes = [] + for key in regular_tl_keys: + all_regular_tl_boxes.extend(self._find_template_raw(key)) + + all_regular_br_boxes = [] + for key in regular_br_keys: + all_regular_br_boxes.extend(self._find_template_raw(key)) + + # --- Find Bot Bubble Corners (Raw Coordinates - Single Type) --- + bot_tl_boxes = self._find_template_raw('bot_corner_tl') # Modified + bot_br_boxes = self._find_template_raw('bot_corner_br') # Modified + + # --- Match Regular Bubbles (Any Type TL with Any Type BR) --- + if all_regular_tl_boxes and all_regular_br_boxes: + for tl_box in all_regular_tl_boxes: + tl_coords = (tl_box[0], tl_box[1]) # Extract original TL (left, top) + # Skip if this TL is already part of a matched bubble + if tl_coords in processed_tls: continue + + potential_br_box = None min_dist_sq = float('inf') - for j, br in enumerate(br_corners): - if br[0] > tl[0] + 20 and br[1] > tl[1] + 10: - dist_sq = (br[0] - tl[0])**2 + (br[1] - tl[1])**2 + # Find the closest valid BR corner (from any regular type) below and to the right + for br_box in all_regular_br_boxes: + br_coords = (br_box[0], br_box[1]) # BR top-left + if br_coords[0] > tl_coords[0] + 20 and br_coords[1] > tl_coords[1] + 10: # Basic geometric check + dist_sq = (br_coords[0] - tl_coords[0])**2 + (br_coords[1] - tl_coords[1])**2 if dist_sq < min_dist_sq: - potential_br = br + potential_br_box = br_box min_dist_sq = dist_sq - if potential_br: - bubble_bbox = (tl[0], tl[1], potential_br[0], potential_br[1]) - all_bubbles_with_type.append((bubble_bbox, False)) - processed_tls.add(i) - # Match Bot bubbles - processed_bot_tls = set() - if bot_tl_corners and bot_br_corners: - for i, tl in enumerate(bot_tl_corners): - if i in processed_bot_tls: continue - potential_br = None - min_dist_sq = float('inf') - for j, br in enumerate(bot_br_corners): - if br[0] > tl[0] + 20 and br[1] > tl[1] + 10: - dist_sq = (br[0] - tl[0])**2 + (br[1] - tl[1])**2 - if dist_sq < min_dist_sq: - potential_br = br - min_dist_sq = dist_sq - if potential_br: - bubble_bbox = (tl[0], tl[1], potential_br[0], potential_br[1]) - all_bubbles_with_type.append((bubble_bbox, True)) - processed_bot_tls.add(i) + if potential_br_box: + # Calculate bbox using TL's top-left and BR's bottom-right + bubble_bbox = (tl_coords[0], tl_coords[1], + potential_br_box[0] + potential_br_box[2], potential_br_box[1] + potential_br_box[3]) + all_bubbles_info.append({ + 'bbox': bubble_bbox, + 'is_bot': False, + 'tl_coords': tl_coords # Store the original TL coords + }) + processed_tls.add(tl_coords) # Mark this TL as used - return all_bubbles_with_type + # --- Match Bot Bubbles (Single Type) --- + if bot_tl_boxes and bot_br_boxes: + for tl_box in bot_tl_boxes: + tl_coords = (tl_box[0], tl_box[1]) # Extract original TL (left, top) + # Skip if this TL is already part of a matched bubble + if tl_coords in processed_tls: continue + + potential_br_box = None + min_dist_sq = float('inf') + # Find the closest valid BR corner below and to the right + for br_box in bot_br_boxes: + br_coords = (br_box[0], br_box[1]) # BR top-left + if br_coords[0] > tl_coords[0] + 20 and br_coords[1] > tl_coords[1] + 10: # Basic geometric check + dist_sq = (br_coords[0] - tl_coords[0])**2 + (br_coords[1] - tl_coords[1])**2 + if dist_sq < min_dist_sq: + potential_br_box = br_box + min_dist_sq = dist_sq + + if potential_br_box: + # Calculate bbox using TL's top-left and BR's bottom-right + bubble_bbox = (tl_coords[0], tl_coords[1], + potential_br_box[0] + potential_br_box[2], potential_br_box[1] + potential_br_box[3]) + all_bubbles_info.append({ + 'bbox': bubble_bbox, + 'is_bot': True, + 'tl_coords': tl_coords # Store the original TL coords + }) + processed_tls.add(tl_coords) # Mark this TL as used + + # Note: This logic prioritizes matching regular bubbles first, then bot bubbles. + # Confidence thresholds might need tuning. + return all_bubbles_info def find_keyword_in_region(self, region: Tuple[int, int, int, int]) -> Optional[Tuple[int, int]]: - """Look for keywords within a specified region.""" + """Look for keywords within a specified region. Returns center coordinates.""" if region[2] <= 0 or region[3] <= 0: return None # Invalid region width/height - # Try lowercase - locations_lower = self._find_template('keyword_wolf_lower', region=region) + # Try original lowercase with grayscale matching + locations_lower = self._find_template('keyword_wolf_lower', region=region, grayscale=True) if locations_lower: - print(f"Found keyword (lowercase) in region {region}, position: {locations_lower[0]}") + print(f"Found keyword (lowercase, grayscale) in region {region}, position: {locations_lower[0]}") return locations_lower[0] - # Try uppercase - locations_upper = self._find_template('keyword_wolf_upper', region=region) + # Try original uppercase with grayscale matching + locations_upper = self._find_template('keyword_wolf_upper', region=region, grayscale=True) if locations_upper: - print(f"Found keyword (uppercase) in region {region}, position: {locations_upper[0]}") + print(f"Found keyword (uppercase, grayscale) in region {region}, position: {locations_upper[0]}") return locations_upper[0] + # Try type3 lowercase (white text, no grayscale) + locations_lower_type3 = self._find_template('keyword_wolf_lower_type3', region=region, grayscale=False) # Added type3 check + if locations_lower_type3: + print(f"Found keyword (lowercase, type3) in region {region}, position: {locations_lower_type3[0]}") + return locations_lower_type3[0] + + # Try type3 uppercase (white text, no grayscale) + locations_upper_type3 = self._find_template('keyword_wolf_upper_type3', region=region, grayscale=False) # Added type3 check + if locations_upper_type3: + print(f"Found keyword (uppercase, type3) in region {region}, position: {locations_upper_type3[0]}") + return locations_upper_type3[0] + return None - def calculate_avatar_coords(self, bubble_bbox: Tuple[int, int, int, int], offset_x: int = AVATAR_OFFSET_X) -> Tuple[int, int]: - """Calculate avatar coordinates based on bubble top-left.""" - tl_x, tl_y = bubble_bbox[0], bubble_bbox[1] + def calculate_avatar_coords(self, bubble_tl_coords: Tuple[int, int], offset_x: int = AVATAR_OFFSET_X) -> Tuple[int, int]: + """ + Calculate avatar coordinates based on the EXACT top-left corner coordinates of the bubble. + Uses the Y-coordinate of the TL corner directly. + """ + tl_x, tl_y = bubble_tl_coords[0], bubble_tl_coords[1] avatar_x = tl_x + offset_x - avatar_y = tl_y # Assuming Y is same as top-left - # print(f"Calculated avatar coordinates: ({int(avatar_x)}, {int(avatar_y)})") # Reduce noise + avatar_y = tl_y # Use the exact Y from the detected TL corner + # print(f"Calculated avatar coordinates using TL {bubble_tl_coords}: ({int(avatar_x)}, {int(avatar_y)})") # Reduce noise return (int(avatar_x), int(avatar_y)) def get_current_ui_state(self) -> str: @@ -277,7 +414,7 @@ class InteractionModule: time.sleep(0.1) self.click_at(coords[0], coords[1]) - time.sleep(0.2) # Wait for menu/reaction + time.sleep(0.1) # Wait for menu/reaction copied = False # Try finding "Copy" menu item first @@ -292,7 +429,7 @@ class InteractionModule: print("'Copy' menu item not found. Attempting Ctrl+C.") try: self.hotkey('ctrl', 'c') - time.sleep(0.2) + time.sleep(0.1) print("Simulated Ctrl+C.") copied = True except Exception as e_ctrlc: @@ -323,7 +460,7 @@ class InteractionModule: try: # 1. Click avatar self.click_at(avatar_coords[0], avatar_coords[1]) - time.sleep(0.3) # Wait for profile card + time.sleep(0.1) # Wait for profile card # 2. Find and click profile option profile_option_locations = self.detector._find_template('profile_option', confidence=0.7) @@ -332,7 +469,7 @@ class InteractionModule: return None # Fail early if critical step missing self.click_at(profile_option_locations[0][0], profile_option_locations[0][1]) print("Clicked user details option.") - time.sleep(0.3) # Wait for user details window + time.sleep(0.1) # Wait for user details window # 3. Find and click "Copy Name" button copy_name_locations = self.detector._find_template('copy_name_button', confidence=0.7) @@ -385,14 +522,14 @@ class InteractionModule: # Click input, paste, send self.click_at(input_coords[0], input_coords[1]) - time.sleep(0.3) + time.sleep(0.1) print("Pasting response...") self.set_clipboard(reply_text) time.sleep(0.1) try: self.hotkey('ctrl', 'v') - time.sleep(0.5) + time.sleep(0.1) print("Pasted.") except Exception as e: print(f"Error pasting response: {e}") @@ -412,12 +549,175 @@ class InteractionModule: try: self.press_key('enter') print("Pressed Enter.") - time.sleep(0.5) + time.sleep(0.1) return True except Exception as e_enter: print(f"Error pressing Enter: {e_enter}") return False +# ============================================================================== +# Position Removal Logic +# ============================================================================== +def remove_user_position(detector: DetectionModule, interactor: InteractionModule, trigger_bubble_region: Tuple[int, int, int, int]) -> bool: + """ + Performs the sequence of UI actions to remove a user's position based on the triggering chat bubble. + Returns True if successful, False otherwise. + """ + print(f"\n--- Starting Position Removal Process (Trigger Bubble Region: {trigger_bubble_region}) ---") + bubble_x, bubble_y, bubble_w, bubble_h = trigger_bubble_region # This is the BBOX, y is top + + # 1. Find the closest position icon above the bubble + search_region_y_end = bubble_y + search_region_y_start = max(0, bubble_y - 55) # Search 55 pixels above + search_region_x_start = max(0, bubble_x - 100) # Search wider horizontally + search_region_x_end = bubble_x + bubble_w + 100 + search_region = (search_region_x_start, search_region_y_start, search_region_x_end - search_region_x_start, search_region_y_end - search_region_y_start) + print(f"Searching for position icons in region: {search_region}") + + position_templates = { + 'DEVELOPMENT': POS_DEV_IMG, 'INTERIOR': POS_INT_IMG, 'SCIENCE': POS_SCI_IMG, + 'SECURITY': POS_SEC_IMG, 'STRATEGY': POS_STR_IMG + } + found_positions = [] + for name, path in position_templates.items(): + # Use unique keys for detector templates + locations = detector._find_template(name.lower() + '_pos', confidence=0.75, region=search_region) + for loc in locations: + found_positions.append({'name': name, 'coords': loc, 'path': path}) + + if not found_positions: + print("Error: No position icons found near the trigger bubble.") + return False + + # Find the closest one to the bubble's top-center + bubble_top_center_x = bubble_x + bubble_w // 2 + bubble_top_center_y = bubble_y + closest_position = min(found_positions, key=lambda p: + (p['coords'][0] - bubble_top_center_x)**2 + (p['coords'][1] - bubble_top_center_y)**2) + + target_position_name = closest_position['name'] + print(f"Found pending position: |{target_position_name}| at {closest_position['coords']}") + + # 2. Click user avatar (offset from bubble top-left) + # IMPORTANT: Use the bubble_y (top of the bbox) for the click Y coordinate. + # The AVATAR_OFFSET_X handles the horizontal positioning relative to the bubble's left edge (bubble_x). + avatar_click_x = bubble_x + AVATAR_OFFSET_X # Use constant offset + avatar_click_y = bubble_y # Use the top Y coordinate of the bubble's bounding box + print(f"Clicking avatar at estimated position: ({avatar_click_x}, {avatar_click_y}) based on bubble top-left ({bubble_x}, {bubble_y})") + interactor.click_at(avatar_click_x, avatar_click_y) + time.sleep(0.15) # Wait for profile page + + # 3. Verify Profile Page and Click Capitol Button + if not detector._find_template('profile_page', confidence=detector.state_confidence): + print("Error: Failed to verify Profile Page after clicking avatar.") + perform_state_cleanup(detector, interactor) # Attempt cleanup + return False + print("Profile page verified.") + + capitol_button_locs = detector._find_template('capitol_button', confidence=0.8) + if not capitol_button_locs: + print("Error: Capitol button (#11) not found on profile page.") + perform_state_cleanup(detector, interactor) + return False + interactor.click_at(capitol_button_locs[0][0], capitol_button_locs[0][1]) + print("Clicked Capitol button.") + time.sleep(0.15) # Wait for capitol page + + # 4. Verify Capitol Page + if not detector._find_template('president_title', confidence=detector.state_confidence): + print("Error: Failed to verify Capitol Page (President Title not found).") + perform_state_cleanup(detector, interactor) + return False + print("Capitol page verified.") + + # 5. Find and Click Corresponding Position Button + position_button_templates = { + 'DEVELOPMENT': 'pos_btn_dev', 'INTERIOR': 'pos_btn_int', 'SCIENCE': 'pos_btn_sci', + 'SECURITY': 'pos_btn_sec', 'STRATEGY': 'pos_btn_str' + } + target_button_key = position_button_templates.get(target_position_name) + if not target_button_key: + print(f"Error: Internal error - unknown position name '{target_position_name}'") + perform_state_cleanup(detector, interactor) + return False + + pos_button_locs = detector._find_template(target_button_key, confidence=0.8) + if not pos_button_locs: + print(f"Error: Position button for '{target_position_name}' not found on Capitol page.") + perform_state_cleanup(detector, interactor) + return False + interactor.click_at(pos_button_locs[0][0], pos_button_locs[0][1]) + print(f"Clicked '{target_position_name}' position button.") + time.sleep(0.15) # Wait for position page + + # 6. Verify Position Page + position_page_templates = { + 'DEVELOPMENT': 'page_dev', 'INTERIOR': 'page_int', 'SCIENCE': 'page_sci', + 'SECURITY': 'page_sec', 'STRATEGY': 'page_str' + } + target_page_key = position_page_templates.get(target_position_name) + if not target_page_key: + print(f"Error: Internal error - unknown position name '{target_position_name}' for page verification") + perform_state_cleanup(detector, interactor) + return False + + if not detector._find_template(target_page_key, confidence=detector.state_confidence): + print(f"Error: Failed to verify correct position page for '{target_position_name}'.") + perform_state_cleanup(detector, interactor) + return False + print(f"Verified '{target_position_name}' position page.") + + # 7. Find and Click Dismiss Button + dismiss_locs = detector._find_template('dismiss_button', confidence=0.8) + if not dismiss_locs: + print("Error: Dismiss button not found on position page.") + perform_state_cleanup(detector, interactor) + return False + interactor.click_at(dismiss_locs[0][0], dismiss_locs[0][1]) + print("Clicked Dismiss button.") + time.sleep(0.1) # Wait for confirmation + + # 8. Find and Click Confirm Button + confirm_locs = detector._find_template('confirm_button', confidence=0.8) + if not confirm_locs: + print("Error: Confirm button not found after clicking dismiss.") + # Don't cleanup here, might be stuck in confirmation state + return False # Indicate failure, but let main loop decide next step + interactor.click_at(confirm_locs[0][0], confirm_locs[0][1]) + print("Clicked Confirm button. Position should be dismissed.") + time.sleep(0.1) # Wait for action to complete + + # 9. Cleanup: Return to Chat Room + # Click Close on position page (should now be back on capitol page implicitly) + close_locs = detector._find_template('close_button', confidence=0.8) + if close_locs: + interactor.click_at(close_locs[0][0], close_locs[0][1]) + print("Clicked Close button (returning to Capitol).") + time.sleep(0.15) + else: + print("Warning: Close button not found after confirm, attempting back arrow anyway.") + + # Click Back Arrow on Capitol page (should return to profile) + back_arrow_locs = detector._find_template('back_arrow', confidence=0.8) + if back_arrow_locs: + interactor.click_at(back_arrow_locs[0][0], back_arrow_locs[0][1]) + print("Clicked Back Arrow (returning to Profile).") + time.sleep(0.15) + else: + print("Warning: Back arrow not found on Capitol page, attempting ESC cleanup.") + + # Use standard ESC cleanup + print("Initiating final ESC cleanup to return to chat...") + cleanup_success = perform_state_cleanup(detector, interactor) + + if cleanup_success: + print("--- Position Removal Process Completed Successfully ---") + return True + else: + print("--- Position Removal Process Completed, but failed to confirm return to chat room ---") + return False # Technically removed, but UI state uncertain + + # ============================================================================== # Coordinator Logic (Placeholder - To be implemented in main.py) # ============================================================================== @@ -432,7 +732,7 @@ def perform_state_cleanup(detector: DetectionModule, interactor: InteractionModu returned_to_chat = False for attempt in range(max_attempts): print(f"Cleanup attempt #{attempt + 1}/{max_attempts}") - time.sleep(0.2) + time.sleep(0.1) current_state = detector.get_current_ui_state() print(f"Detected state: {current_state}") @@ -444,14 +744,14 @@ def perform_state_cleanup(detector: DetectionModule, interactor: InteractionModu elif current_state == 'user_details' or current_state == 'profile_card': print(f"{current_state.replace('_', ' ').title()} detected, pressing ESC...") interactor.press_key('esc') - time.sleep(0.3) # Wait longer for UI response after ESC + time.sleep(0.1) # Wait longer for UI response after ESC continue else: # Unknown state print("Unknown page state detected.") if attempt < max_attempts - 1: print("Trying one ESC press as fallback...") interactor.press_key('esc') - time.sleep(0.3) + time.sleep(0.1) else: print("Maximum attempts reached, stopping cleanup.") break @@ -473,26 +773,64 @@ def run_ui_monitoring_loop(trigger_queue: queue.Queue, command_queue: queue.Queu # Load templates directly using constants defined in this file for now # Consider passing config or a template loader object in the future templates = { + # Regular Bubble (Original + Skins) - Keys match those used in find_dialogue_bubbles 'corner_tl': CORNER_TL_IMG, 'corner_br': CORNER_BR_IMG, + 'corner_tl_type2': CORNER_TL_TYPE2_IMG, 'corner_br_type2': CORNER_BR_TYPE2_IMG, + 'corner_tl_type3': CORNER_TL_TYPE3_IMG, 'corner_br_type3': CORNER_BR_TYPE3_IMG, # Corrected: Added missing keys here + # Bot Bubble (Single Type) 'bot_corner_tl': BOT_CORNER_TL_IMG, 'bot_corner_br': BOT_CORNER_BR_IMG, - 'keyword_wolf_lower': KEYWORD_wolf_LOWER_IMG, 'keyword_wolf_upper': KEYWORD_Wolf_UPPER_IMG, + # Keywords & UI Elements + 'keyword_wolf_lower': KEYWORD_wolf_LOWER_IMG, + 'keyword_wolf_upper': KEYWORD_Wolf_UPPER_IMG, + 'keyword_wolf_lower_type3': KEYWORD_wolf_LOWER_TYPE3_IMG, # Added + 'keyword_wolf_upper_type3': KEYWORD_Wolf_UPPER_TYPE3_IMG, # Added 'copy_menu_item': COPY_MENU_ITEM_IMG, 'profile_option': PROFILE_OPTION_IMG, 'copy_name_button': COPY_NAME_BUTTON_IMG, 'send_button': SEND_BUTTON_IMG, 'chat_input': CHAT_INPUT_IMG, 'profile_name_page': PROFILE_NAME_PAGE_IMG, 'profile_page': PROFILE_PAGE_IMG, 'chat_room': CHAT_ROOM_IMG, - 'world_chat': WORLD_CHAT_IMG, 'private_chat': PRIVATE_CHAT_IMG # Add other templates as needed + 'base_screen': BASE_SCREEN_IMG, 'world_map_screen': WORLD_MAP_IMG, # Added for navigation + 'world_chat': WORLD_CHAT_IMG, 'private_chat': PRIVATE_CHAT_IMG, + # Add position templates + 'development_pos': POS_DEV_IMG, 'interior_pos': POS_INT_IMG, 'science_pos': POS_SCI_IMG, + 'security_pos': POS_SEC_IMG, 'strategy_pos': POS_STR_IMG, + # Add capitol templates + 'capitol_button': CAPITOL_BUTTON_IMG, 'president_title': PRESIDENT_TITLE_IMG, + 'pos_btn_dev': POS_BTN_DEV_IMG, 'pos_btn_int': POS_BTN_INT_IMG, 'pos_btn_sci': POS_BTN_SCI_IMG, + 'pos_btn_sec': POS_BTN_SEC_IMG, 'pos_btn_str': POS_BTN_STR_IMG, + 'page_dev': PAGE_DEV_IMG, 'page_int': PAGE_INT_IMG, 'page_sci': PAGE_SCI_IMG, + 'page_sec': PAGE_SEC_IMG, 'page_str': PAGE_STR_IMG, + 'dismiss_button': DISMISS_BUTTON_IMG, 'confirm_button': CONFIRM_BUTTON_IMG, + 'close_button': CLOSE_BUTTON_IMG, 'back_arrow': BACK_ARROW_IMG } # Use default confidence/region settings from constants detector = DetectionModule(templates, confidence=CONFIDENCE_THRESHOLD, state_confidence=STATE_CONFIDENCE_THRESHOLD, region=SCREENSHOT_REGION) # Use default input coords/keys from constants interactor = InteractionModule(detector, input_coords=(CHAT_INPUT_CENTER_X, CHAT_INPUT_CENTER_Y), input_template_key='chat_input', send_button_key='send_button') - # --- State Management (Local to this monitoring thread) --- - last_processed_bubble_bbox = None +# --- State Management (Local to this monitoring thread) --- + last_processed_bubble_info = None # Store the whole dict now recent_texts = collections.deque(maxlen=RECENT_TEXT_HISTORY_MAXLEN) # Context-specific history needed + screenshot_counter = 0 # Initialize counter for debug screenshots while True: - # --- Process Commands First (Non-blocking) --- + # --- Check for Main Screen Navigation First --- + try: + base_locs = detector._find_template('base_screen', confidence=0.8) + map_locs = detector._find_template('world_map_screen', confidence=0.8) + if base_locs or map_locs: + print("UI Thread: Detected main screen (Base or World Map). Clicking to return to chat...") + # Coordinates provided by user (adjust if needed based on actual screen resolution/layout) + # IMPORTANT: Ensure these coordinates are correct for the target window/resolution + target_x, target_y = 600, 1300 + interactor.click_at(target_x, target_y) + time.sleep(0.2) # Short delay after click + print("UI Thread: Clicked to return to chat. Re-checking screen state...") + continue # Skip the rest of the loop and re-evaluate + except Exception as nav_err: + print(f"UI Thread: Error during main screen navigation check: {nav_err}") + # Decide if you want to continue or pause after error + + # --- Process Commands Second (Non-blocking) --- try: command_data = command_queue.get_nowait() # Check for commands without blocking action = command_data.get('action') @@ -503,6 +841,16 @@ def run_ui_monitoring_loop(trigger_queue: queue.Queue, command_queue: queue.Queu interactor.send_chat_message(text_to_send) else: print("UI Thread: Received send_reply command with no text.") + elif action == 'remove_position': # <--- Handle new command + region = command_data.get('trigger_bubble_region') + if region: + print(f"UI Thread: Received command to remove position triggered by bubble region: {region}") + # Call the new UI function + success = remove_user_position(detector, interactor, region) # Call synchronous function + print(f"UI Thread: Position removal attempt finished. Success: {success}") + # Note: No need to send result back unless main thread needs confirmation + else: + print("UI Thread: Received remove_position command without trigger_bubble_region.") else: print(f"UI Thread: Received unknown command: {action}") except queue.Empty: @@ -510,49 +858,100 @@ def run_ui_monitoring_loop(trigger_queue: queue.Queue, command_queue: queue.Queu except Exception as cmd_err: print(f"UI Thread: Error processing command queue: {cmd_err}") - # --- Then Perform UI Monitoring --- + # --- Verify Chat Room State Before Bubble Detection --- + try: + # Use a slightly lower confidence maybe, or state_confidence + chat_room_locs = detector._find_template('chat_room', confidence=detector.state_confidence) + if not chat_room_locs: + print("UI Thread: Not in chat room state before bubble detection. Attempting cleanup...") + # Call the existing cleanup function to try and return + perform_state_cleanup(detector, interactor) + # Regardless of cleanup success, restart the loop to re-evaluate state from the top + print("UI Thread: Continuing loop after attempting chat room cleanup.") + time.sleep(0.5) # Small pause after cleanup attempt + continue + # else: # Optional: Log if chat room is confirmed + # print("UI Thread: Chat room state confirmed.") + + except Exception as state_check_err: + print(f"UI Thread: Error checking for chat room state: {state_check_err}") + # Decide how to handle error - maybe pause and retry? For now, continue cautiously. + time.sleep(1) + + + # --- Then Perform UI Monitoring (Bubble Detection) --- try: # 1. Detect Bubbles - all_bubbles = detector.find_dialogue_bubbles() - if not all_bubbles: time.sleep(2); continue + all_bubbles_data = detector.find_dialogue_bubbles() # Returns list of dicts + if not all_bubbles_data: time.sleep(2); continue # Filter out bot bubbles, find newest non-bot bubble (example logic) - other_bubbles = [bbox for bbox, is_bot in all_bubbles if not is_bot] - if not other_bubbles: time.sleep(2); continue + other_bubbles_data = [b_info for b_info in all_bubbles_data if not b_info['is_bot']] + if not other_bubbles_data: time.sleep(0.2); continue # Simple logic: assume lowest bubble is newest (might need improvement) - target_bubble = max(other_bubbles, key=lambda b: b[3]) # b[3] is y_max + # Sort by bbox bottom y-coordinate (index 3) + target_bubble_info = max(other_bubbles_data, key=lambda b_info: b_info['bbox'][3]) # 2. Check for Duplicates (Position & Content) - if are_bboxes_similar(target_bubble, last_processed_bubble_bbox): - time.sleep(2); continue + # Compare using the 'bbox' from the info dicts + if are_bboxes_similar(target_bubble_info.get('bbox'), last_processed_bubble_info.get('bbox') if last_processed_bubble_info else None): + time.sleep(0.2); continue # 3. Detect Keyword in Bubble - bubble_region = (target_bubble[0], target_bubble[1], target_bubble[2]-target_bubble[0], target_bubble[3]-target_bubble[1]) + target_bbox = target_bubble_info['bbox'] + bubble_region = (target_bbox[0], target_bbox[1], target_bbox[2]-target_bbox[0], target_bbox[3]-target_bbox[1]) keyword_coords = detector.find_keyword_in_region(bubble_region) if keyword_coords: - print(f"\n!!! Keyword detected in bubble {target_bubble} !!!") + print(f"\n!!! Keyword detected in bubble {target_bbox} !!!") + + # --- Debug Screenshot Logic --- + try: + screenshot_index = (screenshot_counter % MAX_DEBUG_SCREENSHOTS) + 1 + screenshot_filename = f"debug_bubble_{screenshot_index}.png" + screenshot_path = os.path.join(DEBUG_SCREENSHOT_DIR, screenshot_filename) + # --- Enhanced Logging --- + print(f"Attempting to save debug screenshot to: {screenshot_path}") + print(f"Screenshot Region: {bubble_region}") + # Check if region is valid before attempting screenshot + if bubble_region and len(bubble_region) == 4 and bubble_region[2] > 0 and bubble_region[3] > 0: + # Convert numpy types to standard Python int for pyautogui + region_int = (int(bubble_region[0]), int(bubble_region[1]), int(bubble_region[2]), int(bubble_region[3])) + pyautogui.screenshot(region=region_int, imageFilename=screenshot_path) + print(f"Successfully saved debug screenshot: {screenshot_path}") + screenshot_counter += 1 + else: + print(f"Error: Invalid screenshot region {bubble_region}. Skipping screenshot.") + # --- End Enhanced Logging --- + except Exception as ss_err: + # --- Enhanced Error Logging --- + print(f"Error taking debug screenshot for region {bubble_region} to path {screenshot_path}: {repr(ss_err)}") + import traceback + traceback.print_exc() # Print full traceback for detailed debugging + # --- End Enhanced Error Logging --- + # --- End Debug Screenshot Logic --- # 4. Interact: Get Bubble Text bubble_text = interactor.copy_text_at(keyword_coords) if not bubble_text: print("Error: Could not get dialogue content.") - last_processed_bubble_bbox = target_bubble # Mark as processed even if failed + last_processed_bubble_info = target_bubble_info # Mark as processed even if failed perform_state_cleanup(detector, interactor) # Attempt cleanup after failed copy continue # Check recent text history (needs context awareness) if bubble_text in recent_texts: print(f"Content '{bubble_text[:30]}...' in recent history, skipping.") - last_processed_bubble_bbox = target_bubble + last_processed_bubble_info = target_bubble_info continue print(">>> New trigger event <<<") - last_processed_bubble_bbox = target_bubble + last_processed_bubble_info = target_bubble_info recent_texts.append(bubble_text) # 5. Interact: Get Sender Name - avatar_coords = detector.calculate_avatar_coords(target_bubble) + # *** Use the precise TL coordinates for avatar calculation *** + avatar_coords = detector.calculate_avatar_coords(target_bubble_info['tl_coords']) sender_name = interactor.retrieve_sender_name_interaction(avatar_coords) # 6. Perform Cleanup (Crucial after potentially leaving chat screen) @@ -569,15 +968,22 @@ def run_ui_monitoring_loop(trigger_queue: queue.Queue, command_queue: queue.Queu print("\n>>> Putting trigger info in Queue <<<") print(f" Sender: {sender_name}") print(f" Content: {bubble_text[:100]}...") + print(f" Bubble Region: {bubble_region}") # Include region derived from bbox try: - data_to_send = {'sender': sender_name, 'text': bubble_text} + # Include bubble_region in the data sent + data_to_send = { + 'sender': sender_name, + 'text': bubble_text, + 'bubble_region': bubble_region # Use bbox-derived region for general use + # 'tl_coords': target_bubble_info['tl_coords'] # Optionally send if needed elsewhere + } trigger_queue.put(data_to_send) # Put in the queue for main loop - print("Trigger info placed in Queue.") + print("Trigger info (with region) placed in Queue.") except Exception as q_err: print(f"Error putting data in Queue: {q_err}") print("--- Single trigger processing complete ---") - time.sleep(1) # Pause after successful trigger + time.sleep(0.1) # Pause after successful trigger time.sleep(1.5) # Polling interval @@ -591,8 +997,8 @@ def run_ui_monitoring_loop(trigger_queue: queue.Queue, command_queue: queue.Queu # Attempt cleanup in case of unexpected error during interaction print("Attempting cleanup after unexpected error...") perform_state_cleanup(detector, interactor) - print("Waiting 5 seconds before retry...") - time.sleep(5) + print("Waiting 3 seconds before retry...") + time.sleep(3) # Note: The old monitor_chat_for_trigger function is replaced by the example_coordinator_loop. # The actual UI monitoring thread started in main.py should call a function like this example loop.