From 3ec4017a1efae2c0cf1adf8fc58b6157d595a879 Mon Sep 17 00:00:00 2001 From: z060142 Date: Sat, 19 Apr 2025 01:37:57 +0800 Subject: [PATCH] Add Reply to specific conversation. Fix Conversation bubble detection --- ClaudeCode.md | 109 +++++- main.py | 80 +++- templates/reply_button.png | Bin 0 -> 2863 bytes ui_interaction.py | 740 ++++++++++++++++++++++++++++++------- 4 files changed, 771 insertions(+), 158 deletions(-) create mode 100644 templates/reply_button.png diff --git a/ClaudeCode.md b/ClaudeCode.md index 42bb7a4..85484a8 100644 --- a/ClaudeCode.md +++ b/ClaudeCode.md @@ -75,11 +75,26 @@ Wolf Chat 是一個基於 MCP (Modular Capability Provider) 框架的聊天機 系統使用基於圖像辨識的方法監控遊戲聊天界面： -1. **泡泡檢測**：通過辨識聊天泡泡的左上角 (TL) 和右下角 (BR) 角落圖案定位聊天訊息。系統能區分一般用戶泡泡和機器人泡泡。**為了適應玩家可能使用的不同聊天泡泡外觀 (skin)，一般用戶泡泡的偵測機制已被擴充，可以同時尋找多組不同的角落模板 (例如 `corner_tl_type2.png`, `corner_br_type2.png` 等)，提高了對自訂外觀的兼容性。機器人泡泡目前僅偵測預設的角落模板。** -2. **關鍵字檢測**：在泡泡區域內搜尋 "wolf" 或 "Wolf" 關鍵字圖像 -3. **內容獲取**：點擊關鍵字位置，使用剪貼板複製聊天內容 -4. **發送者識別**：**關鍵步驟** - 系統會根據**偵測到關鍵字的那個特定聊天泡泡**的左上角座標，計算出頭像的點擊位置（目前水平偏移量為 -55 像素）。這確保了點擊的是觸發訊息的發送者頭像，而不是其他位置的頭像。接著通過點擊計算出的頭像位置，導航菜單，最終複製用戶名稱。 -5. **防重複處理**：使用位置比較和內容歷史記錄防止重複回應 +1. **泡泡檢測（含 Y 軸優先配對）**：通過辨識聊天泡泡的左上角 (TL) 和右下角 (BR) 角落圖案定位聊天訊息。 + - **多外觀支援**：為了適應玩家可能使用的不同聊天泡泡外觀 (skin)，一般用戶泡泡的偵測機制已被擴充，可以同時尋找多組不同的角落模板 (例如 `corner_tl_type2.png`, `corner_br_type2.png` 等)。機器人泡泡目前僅偵測預設的角落模板。 + - **配對邏輯優化**：在配對 TL 和 BR 角落時，系統現在會優先選擇與 TL 角落 **Y 座標最接近** 的有效 BR 角落，以更好地區分垂直堆疊的聊天泡泡。 +2. **關鍵字檢測**：在泡泡區域內搜尋 "wolf" 或 "Wolf" 關鍵字圖像。 +3. **內容獲取**：點擊關鍵字位置，使用剪貼板複製聊天內容。 +4. **發送者識別（含氣泡重新定位與偏移量調整）**：**關鍵步驟** - 為了提高在動態聊天環境下的穩定性，系統在獲取發送者名稱前，會執行以下步驟： + a. **初始偵測**：像之前一樣，根據偵測到的關鍵字定位觸發的聊天泡泡。 + b. **氣泡快照**：擷取該聊天泡泡的圖像快照。 + c. **重新定位**：在點擊頭像前，使用該快照在當前聊天視窗區域內重新搜尋氣泡的最新位置。 + d. **計算座標（新偏移量）**： + - 如果成功重新定位氣泡，則根據找到的**新**左上角座標 (`new_tl_x`, `new_tl_y`)，應用新的偏移量計算頭像點擊位置：`x = new_tl_x - 45` (`AVATAR_OFFSET_X_REPLY`)，`y = new_tl_y + 10` (`AVATAR_OFFSET_Y_REPLY`)。 + - 如果無法重新定位（例如氣泡已滾動出畫面），則跳過此次互動，以避免點擊錯誤位置。 + e. **互動（含重試）**： + - 使用計算出的（新的）頭像位置進行第一次點擊。 + - 檢查是否成功進入個人資料頁面 (`Profile_page.png`)。 + - **如果失敗**：系統會使用步驟 (b) 的氣泡快照，在聊天區域內重新定位氣泡，重新計算頭像座標，然後再次嘗試點擊。此過程最多重複 3 次。 + - **如果成功**（無論是首次嘗試還是重試成功）：繼續導航菜單，最終複製用戶名稱。 + - **如果重試後仍失敗**：放棄獲取該用戶名稱。 + f. **原始偏移量**：原始的 `-55` 像素水平偏移量 (`AVATAR_OFFSET_X`) 仍保留在程式碼中，用於其他不需要重新定位或不同互動邏輯的場景（例如 `remove_user_position` 功能）。 +5. **防重複處理**：使用位置比較和內容歷史記錄防止重複回應。 #### LLM 整合 @@ -112,10 +127,20 @@ Wolf Chat 是一個基於 MCP (Modular Capability Provider) 框架的聊天機 系統使用多種技術實現 UI 自動化： -1. **圖像辨識**：使用 OpenCV 和 pyautogui 進行圖像匹配和識別 -2. **鍵鼠控制**：模擬鼠標點擊和鍵盤操作 -3. **剪貼板操作**：使用 pyperclip 讀寫剪貼板 -4. **狀態式處理**：基於 UI 狀態判斷的互動流程，確保操作穩定性 +1. **圖像辨識**：使用 OpenCV 和 pyautogui 進行圖像匹配和識別。 +2. **鍵鼠控制**：模擬鼠標點擊和鍵盤操作。 +3. **剪貼板操作**：使用 pyperclip 讀寫剪貼板。 +4. **狀態式處理**：基於 UI 狀態判斷的互動流程，確保操作穩定性。 +5. **針對性回覆（上下文激活）**： + - **時機**：在成功獲取發送者名稱並返回聊天介面後，但在將觸發資訊放入隊列傳遞給主線程之前。 + - **流程**： + a. 再次使用氣泡快照重新定位觸發訊息的氣泡。 + b. 如果定位成功，點擊氣泡中心，並等待 0.25 秒（增加的延遲時間）以允許 UI 反應。 + c. 尋找並點擊彈出的「回覆」按鈕 (`reply_button.png`)。 + d. 如果成功點擊回覆按鈕，則設置一個 `reply_context_activated` 標記為 `True`。 + e. 如果重新定位氣泡失敗或未找到回覆按鈕，則該標記為 `False`。 + - **傳遞**：將 `reply_context_activated` 標記連同其他觸發資訊（發送者、內容、氣泡區域）一起放入隊列。 + - **發送**：主控模塊 (`main.py`) 在處理 `send_reply` 命令時，不再需要執行點擊回覆的操作，只需直接調用 `send_chat_message` 即可（因為如果 `reply_context_activated` 為 `True`，輸入框應已準備好）。 ## 配置與部署 @@ -192,6 +217,72 @@ Wolf Chat 是一個基於 MCP (Modular Capability Provider) 框架的聊天機 - 在「技術實現」的「發送者識別」部分強調了點擊位置是相對於觸發泡泡計算的，並註明了新的偏移量。 - 添加了此「最近改進」條目。 +### 聊天泡泡重新定位以提高穩定性 + +- **UI 互動模塊 (`ui_interaction.py`)**： + - 在 `run_ui_monitoring_loop` 中，於偵測到關鍵字並成功複製文字後、獲取發送者名稱前，加入了新的邏輯： + 1. 擷取觸發氣泡的圖像快照。 + 2. 使用 `pyautogui.locateOnScreen` 在聊天區域內重新尋找該快照的當前位置。 + 3. 若找到，則根據**新位置**的左上角座標和新的偏移量 (`AVATAR_OFFSET_X_RELOCATED = -50`) 計算頭像點擊位置。 + 4. 若找不到，則記錄警告並跳過此次互動。 + - 新增了 `AVATAR_OFFSET_X_RELOCATED` 和 `BUBBLE_RELOCATE_CONFIDENCE` 常數。 +- **目的**：解決聊天視窗內容滾動後，原始偵測到的氣泡位置失效，導致點擊錯誤頭像的問題。透過重新定位，確保點擊的是與觸發訊息相對應的頭像。 +- **文件更新 (`ClaudeCode.md`)**： + - 更新了「技術實現」中的「發送者識別」部分，詳細說明了重新定位的步驟。 + - 在此「最近改進」部分添加了這個新條目。 + +### 互動流程優化 (頭像偏移、氣泡配對、針對性回覆) + +- **UI 互動模塊 (`ui_interaction.py`)**： + - **頭像偏移量調整**：修改了重新定位氣泡後計算頭像座標的邏輯，使用新的偏移量：左 `-45` (`AVATAR_OFFSET_X_REPLY`)，下 `+10` (`AVATAR_OFFSET_Y_REPLY`)。原始的 `-55` 偏移量 (`AVATAR_OFFSET_X`) 保留用於其他功能。 + - **氣泡配對優化**：修改 `find_dialogue_bubbles` 函數，使其在配對左上角 (TL) 和右下角 (BR) 時，優先選擇 Y 座標差異最小的 BR 角落，以提高垂直相鄰氣泡的區分度。 + - **頭像點擊重試**：修改 `retrieve_sender_name_interaction` 函數，增加了最多 3 次的重試邏輯。如果在點擊頭像後未能檢測到個人資料頁面，會嘗試重新定位氣泡並再次點擊。 + - **針對性回覆時機調整與延遲增加**： + - 將點擊氣泡中心和回覆按鈕的操作移至成功獲取發送者名稱並返回聊天室之後、將觸發資訊放入隊列之前。 + - **增加了點擊氣泡中心後、尋找回覆按鈕前的等待時間至 0.25 秒**，以提高在 UI 反應較慢時找到按鈕的成功率。 + - 在放入隊列的數據中增加 `reply_context_activated` 標記，指示是否成功激活了回覆上下文。 + - 簡化了處理 `send_reply` 命令的邏輯，使其僅負責發送消息。 + - **氣泡快照保存 (用於除錯)**：在偵測到關鍵字後，擷取用於重新定位的氣泡圖像快照 (`bubble_snapshot`) 時，會將此快照保存到 `debug_screenshots` 文件夾中，檔名格式為 `debug_relocation_snapshot_X.png` (X 為 1 到 5 的循環數字)。這取代了先前僅保存氣泡區域截圖的邏輯。 +- **目的**： + - 進一步提高獲取發送者名稱的穩定性。 + - 改善氣泡配對的準確性。 + - 調整針對性回覆的流程，使其更符合邏輯順序，並通過增加延遲提高可靠性。 + - 提供用於重新定位的實際圖像快照，方便除錯。 +- **文件更新 (`ClaudeCode.md`)**： + - 更新了「技術實現」中的「泡泡檢測」、「發送者識別」部分。 + - 更新了「UI 自動化」部分關於「針對性回覆」的說明，反映了新的時機、標記和增加的延遲。 + - 在此「最近改進」部分更新了這個匯總條目，以包含最新的修改（包括快照保存和延遲增加）。 + +### UI 監控暫停與恢復機制 (2025-04-18) + +- **目的**：解決在等待 LLM 回應期間，持續的 UI 監控可能導致的不穩定性或干擾問題，特別是與 `remove_position` 等需要精確 UI 狀態的操作相關。 +- **`ui_interaction.py`**： + - 引入了全局（模塊級）`monitoring_paused_flag` 列表（包含一個布爾值）。 + - 在 `run_ui_monitoring_loop` 的主循環開始處檢查此標誌。若為 `True`，則循環僅檢查命令隊列中的 `resume` 命令並休眠，跳過所有 UI 偵測和觸發邏輯。 + - 在命令處理邏輯中添加了對 `pause` 和 `resume` 動作的處理，分別設置 `monitoring_paused_flag[0]` 為 `True` 或 `False`。 +- **`ui_interaction.py` (進一步修改)**： + - **修正命令處理邏輯**：修改了 `run_ui_monitoring_loop` 的主循環。現在，在每次迭代開始時，它會使用一個內部 `while True` 循環和 `command_queue.get_nowait()` 來**處理完隊列中所有待處理的命令**（包括 `pause`, `resume`, `send_reply`, `remove_position` 等）。 + - **狀態檢查後置**：只有在清空當前所有命令後，循環才會檢查 `monitoring_paused_flag` 的狀態。如果標誌為 `True`，則休眠並跳過 UI 監控部分；如果為 `False`，則繼續執行 UI 監控（畫面檢查、氣泡偵測等）。 + - **目的**：解決先前版本中 `resume` 命令可能導致 UI 線程過早退出暫停狀態，從而錯過緊隨其後的 `send_reply` 或 `remove_position` 命令的問題。確保所有來自 `main.py` 的命令都被及時處理。 +- **`main.py`**： + - （先前修改保持不變）在主處理循環 (`run_main_with_exit_stack` 的 `while True` 循環) 中： + - 在從 `trigger_queue` 獲取數據後、調用 `llm_interaction.get_llm_response` **之前**，向 `command_queue` 發送 `{ 'action': 'pause' }` 命令。 + - 使用 `try...finally` 結構，確保在處理 LLM 回應（包括命令處理和發送回覆）**之後**，向 `command_queue` 發送 `{ 'action': 'resume' }` 命令，無論處理過程中是否發生錯誤。 + +### `remove_position` 穩定性改進 (使用快照重新定位) (2025-04-19) + +- **目的**：解決 `remove_position` 命令因聊天視窗滾動導致基於舊氣泡位置計算座標而出錯的問題。 +- **`ui_interaction.py` (`run_ui_monitoring_loop`)**： + - 在觸發事件放入 `trigger_queue` 的數據中，額外添加了 `bubble_snapshot`（觸發氣泡的圖像快照）和 `search_area`（用於快照的搜索區域）。 +- **`main.py`**： + - 修改了處理 `remove_position` 命令的邏輯，使其從 `trigger_data` 中提取 `bubble_snapshot` 和 `search_area`，並將它們包含在發送給 `command_queue` 的命令數據中。 +- **`ui_interaction.py` (`remove_user_position` 函數)**： + - 修改了函數簽名，以接收 `bubble_snapshot` 和 `search_area` 參數。 + - 在函數執行開始時，使用傳入的 `bubble_snapshot` 和 `search_area` 調用 `pyautogui.locateOnScreen` 來重新定位觸發氣泡的當前位置。 + - 如果重新定位失敗，則記錄錯誤並返回 `False`。 + - 如果重新定位成功，則後續所有基於氣泡位置的計算（包括尋找職位圖標的搜索區域 `search_region` 和點擊頭像的座標 `avatar_click_x`, `avatar_click_y`）都將使用這個**新找到的**氣泡座標。 +- **效果**：確保 `remove_position` 操作基於氣泡的最新位置執行，提高了在動態滾動的聊天界面中的可靠性。 + ## 開發建議 ### 優化方向 diff --git a/main.py b/main.py index 395b8f7..b0281b1 100644 --- a/main.py +++ b/main.py @@ -225,9 +225,21 @@ async def run_main_with_exit_stack(): # Use run_in_executor to wait for item from standard queue trigger_data = await loop.run_in_executor(None, trigger_queue.get) + # --- Pause UI Monitoring --- + print("Pausing UI monitoring before LLM call...") + pause_command = {'action': 'pause'} + try: + await loop.run_in_executor(None, command_queue.put, pause_command) + print("Pause command placed in queue.") + except Exception as q_err: + print(f"Error putting pause command in queue: {q_err}") + # --- End Pause --- + sender_name = trigger_data.get('sender') bubble_text = trigger_data.get('text') bubble_region = trigger_data.get('bubble_region') # <-- Extract bubble_region + bubble_snapshot = trigger_data.get('bubble_snapshot') # <-- Extract snapshot + search_area = trigger_data.get('search_area') # <-- Extract search_area print(f"\n--- Received trigger from UI ---") print(f" Sender: {sender_name}") print(f" Content: {bubble_text[:100]}...") @@ -262,25 +274,61 @@ async def run_main_with_exit_stack(): cmd_type = cmd.get("type", "") cmd_params = cmd.get("parameters", {}) # Parameters might be empty for remove_position - # --- Command Processing --- +# --- Command Processing --- if cmd_type == "remove_position": if bubble_region: # Check if we have the context - print("Sending 'remove_position' command to UI thread...") - command_to_send = { - 'action': 'remove_position', - 'trigger_bubble_region': bubble_region # Pass the region - } - try: - await loop.run_in_executor(None, command_queue.put, command_to_send) - print("Command placed in queue.") - except Exception as q_err: - print(f"Error putting remove_position command in queue: {q_err}") + # Debug info - print what we have + print(f"Processing remove_position command with:") + print(f" bubble_region: {bubble_region}") + print(f" bubble_snapshot available: {'Yes' if bubble_snapshot is not None else 'No'}") + print(f" search_area available: {'Yes' if search_area is not None else 'No'}") + + # Check if we have snapshot and search_area as well + if bubble_snapshot and search_area: + print("Sending 'remove_position' command to UI thread with snapshot and search area...") + command_to_send = { + 'action': 'remove_position', + 'trigger_bubble_region': bubble_region, # Original region (might be outdated) + 'bubble_snapshot': bubble_snapshot, # Snapshot for re-location + 'search_area': search_area # Area to search in + } + try: + await loop.run_in_executor(None, command_queue.put, command_to_send) + except Exception as q_err: + print(f"Error putting remove_position command in queue: {q_err}") + else: + # If we have bubble_region but missing other parameters, use a dummy search area + # and let UI thread take a new screenshot + print("Missing bubble_snapshot or search_area, trying with defaults...") + + # Use the bubble_region itself as a fallback search area if needed + default_search_area = None + if search_area is None and bubble_region: + # Convert bubble_region to a proper search area format if needed + if len(bubble_region) == 4: + default_search_area = bubble_region + + command_to_send = { + 'action': 'remove_position', + 'trigger_bubble_region': bubble_region, + 'bubble_snapshot': bubble_snapshot, # Pass as is, might be None + 'search_area': default_search_area if search_area is None else search_area + } + + try: + await loop.run_in_executor(None, command_queue.put, command_to_send) + print("Command sent with fallback parameters.") + except Exception as q_err: + print(f"Error putting remove_position command in queue: {q_err}") else: print("Error: Cannot process 'remove_position' command without bubble_region context.") # Add other command handling here if needed # elif cmd_type == "some_other_command": # # Handle other commands # pass + # elif cmd_type == "some_other_command": + # # Handle other commands + # pass else: print(f"Received unhandled command type: {cmd_type}, parameters: {cmd_params}") # --- End Command Processing --- @@ -307,6 +355,16 @@ async def run_main_with_exit_stack(): print(f"\nError processing trigger or sending response: {e}") import traceback traceback.print_exc() + finally: + # --- Resume UI Monitoring --- + print("Resuming UI monitoring after processing...") + resume_command = {'action': 'resume'} + try: + await loop.run_in_executor(None, command_queue.put, resume_command) + print("Resume command placed in queue.") + except Exception as q_err: + print(f"Error putting resume command in queue: {q_err}") + # --- End Resume --- # No task_done needed for standard queue except asyncio.CancelledError: diff --git a/templates/reply_button.png b/templates/reply_button.png new file mode 100644 index 0000000000000000000000000000000000000000..b6a8bd223258441799e219475a9c52f8ffde1cca GIT binary patch literal 2863 zcmV+~3()k5P)KRG%1{F>NoUSMEYco?70<8U~>?)d+O*y)1k#Kh#t$i$v)k5PTWAwq>B zTP{z!1?&Z})19B!w!CWhn$~CEJc*2q@O9sBzS!w@baYKk&U#I&N}a1x<@&nsH(%^@ zfB3#(b@hYSv`S;6Yip~0-S?X>cDn!g`@grgw!Nk`@=nCFXHR_H_nR+vy3e2gxUs(J zHLVg!)Z-_QeBJk7kSr}N@9x^jZ-l}ii9~$OW_{zCZ|Ra{Z*Q+%)3vy$C%;TeN-Qfc z@%Q(?PRn+>4Gqmp`W3JBCq^o%tE*vJ%gVAruU|SjJ#}9#ARs^>;E6;cu{g?W79fkw zW@~w+F&Ng!FXQ5*l~v_jF84Yu+vzqnwd%C`>ooL95*IIh_^^s;E$3;T#`F3anQ8g? zs%skb3c@&DOyKP6 ztGNFK(^{@T7wg=Ur?q$PM0(8>EHBXQ>FFCCnebYFGPBZ(iwl_6as|4O%Br%YB$?Mt z!SVv#-k$!Ek#TYXm+K!LEly39@0d)(LnAh){bypw7=wLcd}?iNopSIH6eLJZO%{cT z7Ik{9cJc7=h+LYLonBOw&$QKO>2m%3^9ocvUZB;w+xPQZ=ikDIWo2iO-|X$}TP&9S z{r#h(BXpC^VsrdB0bDMR&kqR+CU2OZo}Qj<8(fUzUS2zOL) zPxs*9z+$#gK2jJgL@-EG#Ab(whf(Pc3=9vv9i{-t%Tod#`ZGVb(An9GzJ2^+QApK; z3gQ!u#;w<#Jpk+|h3#r;Dx<{r@QgHWcbFr z-`_V#ZsO}_j~&Yo21xTO4SBnxxHumXCWaxNF&;7R2M69cO5@|DHKdRja~Dw?dHUuT zD06eOeMPsrrjpO+_{`c?4C>4~-9CeDDQJ}G6DWly0pE3dE?__=jRuk&wd_0s)~z?C7$hc$3`c*yZgwM>g#JH z(PDxh=r%RCZEhGpez77;tyU0D2~K-x=pA|4juPLktF7_po@L^+Q@^+*KQl9(VYn{o z!s`SD1>qTUVq#*7Qo%8IM@Pnbdip(46cZysn}LCWv~*nwpU)e`T6hAhm;C%OPN_2SV7x0G+AP7 ztkh&OZEx>f|tRR%47)` ztoYs4)e9C!xR8<}x9;v@p+Sg5F8cDT`g`~8x}uw$oQNQ9Wo4D}o)}4t)194}YioPW zw7DGuw;vrNVOq--G9rZfwRJU^g@D)8)Usu~aBfysUXqfMjQF{yyHBUPupVRXB9@`0 zi-laN%H7;FwzYOpR$qlOJ1&W|n_oR8rvwbi73k&{sEBY63ra&n zv(;iHm!@Z=hK7W|jlFp@NcqJ_wbfY5Xz5~1AhF!D?X|RMDAO0I7ZR|WM)%uae=QP) zGOguP>E`FFauxPS-neOe`LfC8@hvmLrnGd+D@rlrkz8T!E-hU+s>#YqFDfozoGuoa zAHHv7+T0HL&2PVj4`f=4E4oPKXQt|rEdCa%`LRHv{Q;`=x%ImKL6K`Oq<&wfBe($5Oy)G#Z^XBQd+cY-EC}qMJY;2Nh+_f z?|yG@Z~xTTLJ6G;3e;jrl=GJ(YYabMXlUtvQC6ImmI8#YnyF!0c6LVLWu959)ipP@ zQQ9Oj?D6qQWGj>+-_!l>FMsfY7FTrdSC+F_r%gY#8jV}zA|$MJ^)u7Fe+ zvn?PXz?DW?y2$L)(^JeAOJf6dJgM`Ph`HR;)#(=TB~HD#hp=>EUQ78x7A;+i#rn6u z{`2&*gYM5F$VUGB-S54i$jx3AkH_on=^q^(r<9&=vz%=*Bgcc&qc~DXTACbwnApZS zIX*c)7g3l9aTCi0UDnxtq|j=$I-A!qt=-UFSvI_Q@ssDhrlAW3a761~xmxA5Yd7w9 zB4=mk+FCm)rHFfBQ(|^@clC~qO?sjTv$MG5?8xg=cQb11s*(4*r)xACG3k*md)Ouo z-NVDf@v+Il!FQCyoX;*WTuEs`aCiR^wnyG^1o+9!^7|2^bcOTV7aH8o13+{Y1HeNDes9q znB~Fw?#hZ`;O!8(LTG46W>z{%+cnfZC?fMFp4sHT0Ysi|He1Xlvte~@dTQ2_Y|qb+ z6L}{hJ}wrk5tr-l@8|LkAIFaas39T2*SzQFxhyW~r>3azghGK*slZUf^VHM~5`79g z6y@iuc|800TV6J-t*-4LdNx`15B3iZ&i1AKI2>-kS$BDXfdXNWSR5sjC9qt_%#o21 z^K%ObU3YhF2)V<;LSy5kGFc)~;o#t4czA@ejTIgqhGF4|&$qXCHr6*RRx6?w3{8}x zOju?2;BST=rXM13gk>0U?RV;uO;5uZi%qa)+4POFF? zUD36+wzjXW($K}g2@(nvifnxGEmw*#L>I|-M|+p^5tFmT#7MAOBi|!_q^FD3rMaoq zxjBJ=FO^1TWoLwiiEhz43v+aluX^%_UWG+@+1Z(nA3dduB^D99EGZGItYc{03PpTN zH#JSJROSeULdP3=y2uvLoMWek$LFP`$&->25s2UZ&Ab}{-JYI4BCaL`p{7c$AZ}e< zS&)?X_VytR+O_Ud(lg))6@}cr6DgL6lapl>hst*KUs3w|2e4=#9UaS(5;8JU;flSJ zr2bC}LTi)Be*oAA0Q`luO_Trt N002ovPDHLkV1nN#ssI20 literal 0 HcmV?d00001 diff --git a/ui_interaction.py b/ui_interaction.py index bbd9db7..497b605 100644 --- a/ui_interaction.py +++ b/ui_interaction.py @@ -13,6 +13,12 @@ import pygetwindow as gw # Used to check/activate windows import config # Used to read window title import queue from typing import List, Tuple, Optional, Dict, Any +import threading # Import threading for Lock if needed, or just use a simple flag + +# --- Global Pause Flag --- +# Using a simple mutable object (list) for thread-safe-like access without explicit lock +# Or could use threading.Event() +monitoring_paused_flag = [False] # List containing a boolean # --- Configuration Section --- SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -21,7 +27,7 @@ os.makedirs(TEMPLATE_DIR, exist_ok=True) # --- Debugging --- DEBUG_SCREENSHOT_DIR = os.path.join(SCRIPT_DIR, "debug_screenshots") -MAX_DEBUG_SCREENSHOTS = 5 +MAX_DEBUG_SCREENSHOTS = 8 os.makedirs(DEBUG_SCREENSHOT_DIR, exist_ok=True) # --- End Debugging --- @@ -94,6 +100,7 @@ DISMISS_BUTTON_IMG = os.path.join(TEMPLATE_DIR, "capitol", "dismiss.png") CONFIRM_BUTTON_IMG = os.path.join(TEMPLATE_DIR, "capitol", "confirm.png") CLOSE_BUTTON_IMG = os.path.join(TEMPLATE_DIR, "capitol", "close_button.png") BACK_ARROW_IMG = os.path.join(TEMPLATE_DIR, "capitol", "black_arrow_down.png") +REPLY_BUTTON_IMG = os.path.join(TEMPLATE_DIR, "reply_button.png") # Added for reply functionality # --- Operation Parameters (Consider moving to config.py) --- @@ -101,9 +108,14 @@ CHAT_INPUT_REGION = None # Example: (100, 800, 500, 50) CHAT_INPUT_CENTER_X = 400 CHAT_INPUT_CENTER_Y = 1280 SCREENSHOT_REGION = None -CONFIDENCE_THRESHOLD = 0.8 +CONFIDENCE_THRESHOLD = 0.9 # Increased threshold for corner matching STATE_CONFIDENCE_THRESHOLD = 0.7 -AVATAR_OFFSET_X = -55 # Adjusted as per user request (was -50) +AVATAR_OFFSET_X = -55 # Original offset, used for non-reply interactions like position removal +# AVATAR_OFFSET_X_RELOCATED = -50 # Replaced by specific reply offsets +AVATAR_OFFSET_X_REPLY = -45 # Horizontal offset for avatar click after re-location (for reply context) +AVATAR_OFFSET_Y_REPLY = 10 # Vertical offset for avatar click after re-location (for reply context) +BUBBLE_RELOCATE_CONFIDENCE = 0.8 # Reduced confidence for finding the bubble snapshot (was 0.9) +BUBBLE_RELOCATE_FALLBACK_CONFIDENCE = 0.6 # Lower confidence for fallback attempts BBOX_SIMILARITY_TOLERANCE = 10 RECENT_TEXT_HISTORY_MAXLEN = 5 # This state likely belongs in the coordinator @@ -237,15 +249,20 @@ class DetectionModule: if tl_coords in processed_tls: continue potential_br_box = None - min_dist_sq = float('inf') - # Find the closest valid BR corner (from any regular type) below and to the right + min_y_diff = float('inf') # Prioritize minimum Y difference + # Find the valid BR corner (from any regular type) with the closest Y-coordinate for br_box in all_regular_br_boxes: br_coords = (br_box[0], br_box[1]) # BR top-left - if br_coords[0] > tl_coords[0] + 20 and br_coords[1] > tl_coords[1] + 10: # Basic geometric check - dist_sq = (br_coords[0] - tl_coords[0])**2 + (br_coords[1] - tl_coords[1])**2 - if dist_sq < min_dist_sq: + # Basic geometric check: BR must be below and to the right of TL + if br_coords[0] > tl_coords[0] + 20 and br_coords[1] > tl_coords[1] + 10: + y_diff = abs(br_coords[1] - tl_coords[1]) # Calculate Y difference + if y_diff < min_y_diff: potential_br_box = br_box - min_dist_sq = dist_sq + min_y_diff = y_diff + # Optional: Add a secondary check for X distance if Y diff is the same? + # elif y_diff == min_y_diff: + # if potential_br_box is None or abs(br_coords[0] - tl_coords[0]) < abs(potential_br_box[0] - tl_coords[0]): + # potential_br_box = br_box if potential_br_box: # Calculate bbox using TL's top-left and BR's bottom-right @@ -266,15 +283,20 @@ class DetectionModule: if tl_coords in processed_tls: continue potential_br_box = None - min_dist_sq = float('inf') - # Find the closest valid BR corner below and to the right + min_y_diff = float('inf') # Prioritize minimum Y difference + # Find the valid BR corner with the closest Y-coordinate for br_box in bot_br_boxes: br_coords = (br_box[0], br_box[1]) # BR top-left - if br_coords[0] > tl_coords[0] + 20 and br_coords[1] > tl_coords[1] + 10: # Basic geometric check - dist_sq = (br_coords[0] - tl_coords[0])**2 + (br_coords[1] - tl_coords[1])**2 - if dist_sq < min_dist_sq: + # Basic geometric check: BR must be below and to the right of TL + if br_coords[0] > tl_coords[0] + 20 and br_coords[1] > tl_coords[1] + 10: + y_diff = abs(br_coords[1] - tl_coords[1]) # Calculate Y difference + if y_diff < min_y_diff: potential_br_box = br_box - min_dist_sq = dist_sq + min_y_diff = y_diff + # Optional: Add a secondary check for X distance if Y diff is the same? + # elif y_diff == min_y_diff: + # if potential_br_box is None or abs(br_coords[0] - tl_coords[0]) < abs(potential_br_box[0] - tl_coords[0]): + # potential_br_box = br_box if potential_br_box: # Calculate bbox using TL's top-left and BR's bottom-right @@ -295,16 +317,16 @@ class DetectionModule: """Look for keywords within a specified region. Returns center coordinates.""" if region[2] <= 0 or region[3] <= 0: return None # Invalid region width/height - # Try original lowercase with grayscale matching - locations_lower = self._find_template('keyword_wolf_lower', region=region, grayscale=True) + # Try original lowercase with color matching + locations_lower = self._find_template('keyword_wolf_lower', region=region, grayscale=False) # Changed grayscale to False if locations_lower: - print(f"Found keyword (lowercase, grayscale) in region {region}, position: {locations_lower[0]}") + print(f"Found keyword (lowercase, color) in region {region}, position: {locations_lower[0]}") # Updated log message return locations_lower[0] - # Try original uppercase with grayscale matching - locations_upper = self._find_template('keyword_wolf_upper', region=region, grayscale=True) + # Try original uppercase with color matching + locations_upper = self._find_template('keyword_wolf_upper', region=region, grayscale=False) # Changed grayscale to False if locations_upper: - print(f"Found keyword (uppercase, grayscale) in region {region}, position: {locations_upper[0]}") + print(f"Found keyword (uppercase, color) in region {region}, position: {locations_upper[0]}") # Updated log message return locations_upper[0] # Try type3 lowercase (white text, no grayscale) @@ -423,7 +445,7 @@ class InteractionModule: copy_coords = copy_item_locations[0] self.click_at(copy_coords[0], copy_coords[1]) print("Clicked 'Copy' menu item.") - time.sleep(0.2) + time.sleep(0.15) copied = True else: print("'Copy' menu item not found. Attempting Ctrl+C.") @@ -446,60 +468,108 @@ class InteractionModule: print("Error: Copy operation unsuccessful or clipboard content invalid.") return None - def retrieve_sender_name_interaction(self, avatar_coords: Tuple[int, int]) -> Optional[str]: + def retrieve_sender_name_interaction(self, + initial_avatar_coords: Tuple[int, int], + bubble_snapshot: Any, # PIL Image object + search_area: Optional[Tuple[int, int, int, int]]) -> Optional[str]: """ Perform the sequence of actions to copy sender name, *without* cleanup. + Includes retries with bubble re-location if the initial avatar click fails. Returns the name or None if failed. """ - print(f"Attempting interaction to get username from avatar {avatar_coords}...") + print(f"Attempting interaction to get username, initial avatar guess: {initial_avatar_coords}...") original_clipboard = self.get_clipboard() or "" self.set_clipboard("___MCP_CLEAR___") time.sleep(0.1) sender_name = None + profile_page_found = False + current_avatar_coords = initial_avatar_coords - try: - # 1. Click avatar - self.click_at(avatar_coords[0], avatar_coords[1]) - time.sleep(0.1) # Wait for profile card + for attempt in range(3): # Retry up to 3 times + print(f"Attempt #{attempt + 1} to click avatar and find profile page...") - # 2. Find and click profile option - profile_option_locations = self.detector._find_template('profile_option', confidence=0.7) - if not profile_option_locations: - print("Error: User details option not found on profile card.") - return None # Fail early if critical step missing - self.click_at(profile_option_locations[0][0], profile_option_locations[0][1]) - print("Clicked user details option.") - time.sleep(0.1) # Wait for user details window + # --- Re-locate bubble on retries --- + if attempt > 0: + print("Re-locating bubble before retry...") + if bubble_snapshot is None: + print("Error: Cannot retry re-location, bubble snapshot is missing.") + break # Cannot retry without snapshot - # 3. Find and click "Copy Name" button - copy_name_locations = self.detector._find_template('copy_name_button', confidence=0.7) - if not copy_name_locations: - print("Error: 'Copy Name' button not found in user details.") - return None # Fail early - self.click_at(copy_name_locations[0][0], copy_name_locations[0][1]) - print("Clicked 'Copy Name' button.") - time.sleep(0.1) + new_bubble_box_retry = pyautogui.locateOnScreen(bubble_snapshot, region=search_area, confidence=BUBBLE_RELOCATE_CONFIDENCE) + if new_bubble_box_retry: + new_tl_x_retry, new_tl_y_retry = new_bubble_box_retry.left, new_bubble_box_retry.top + print(f"Successfully re-located bubble snapshot for retry at: ({new_tl_x_retry}, {new_tl_y_retry})") + # Recalculate avatar coords for the retry + current_avatar_coords = (new_tl_x_retry + AVATAR_OFFSET_X_REPLY, new_tl_y_retry + AVATAR_OFFSET_Y_REPLY) + print(f"Recalculated avatar coordinates for retry: {current_avatar_coords}") + else: + print("Warning: Failed to re-locate bubble snapshot on retry. Aborting name retrieval.") + break # Stop retrying if bubble can't be found - # 4. Get name from clipboard - copied_name = self.get_clipboard() - if copied_name and copied_name != "___MCP_CLEAR___": - print(f"Successfully copied username: {copied_name}") - sender_name = copied_name.strip() + # --- Click Avatar --- + try: + self.click_at(current_avatar_coords[0], current_avatar_coords[1]) + time.sleep(0.15) # Slightly longer wait after click to allow UI to update + except Exception as click_err: + print(f"Error clicking avatar at {current_avatar_coords} on attempt {attempt + 1}: {click_err}") + time.sleep(0.3) # Wait a bit longer after a click error before retrying + continue # Go to next attempt + + # --- Check for Profile Page --- + if self.detector._find_template('profile_page', confidence=self.detector.state_confidence): + print("Profile page verified.") + profile_page_found = True + break # Success, exit retry loop else: - print("Error: Clipboard content invalid after clicking copy name.") - sender_name = None + print(f"Profile page not found after click attempt {attempt + 1}.") + # Optional: Press ESC once to close potential wrong menus before retrying? + # self.press_key('esc') + # time.sleep(0.1) + time.sleep(0.3) # Wait before next attempt - return sender_name + # --- If Profile Page was found, proceed --- + if profile_page_found: + try: + # 2. Find and click profile option + profile_option_locations = self.detector._find_template('profile_option', confidence=0.7) + if not profile_option_locations: + print("Error: User details option not found on profile card.") + return None # Fail early if critical step missing + self.click_at(profile_option_locations[0][0], profile_option_locations[0][1]) + print("Clicked user details option.") + time.sleep(0.1) # Wait for user details window - except Exception as e: - print(f"Error during username retrieval interaction: {e}") - import traceback - traceback.print_exc() - return None - finally: - # Restore clipboard regardless of success/failure - self.set_clipboard(original_clipboard) - # NO cleanup logic here - should be handled by coordinator + # 3. Find and click "Copy Name" button + copy_name_locations = self.detector._find_template('copy_name_button', confidence=0.7) + if not copy_name_locations: + print("Error: 'Copy Name' button not found in user details.") + return None # Fail early + self.click_at(copy_name_locations[0][0], copy_name_locations[0][1]) + print("Clicked 'Copy Name' button.") + time.sleep(0.1) + + # 4. Get name from clipboard + copied_name = self.get_clipboard() + if copied_name and copied_name != "___MCP_CLEAR___": + print(f"Successfully copied username: {copied_name}") + sender_name = copied_name.strip() + else: + print("Error: Clipboard content invalid after clicking copy name.") + sender_name = None + + except Exception as e: + print(f"Error during username retrieval interaction (after profile page found): {e}") + import traceback + traceback.print_exc() + sender_name = None # Ensure None is returned on error + else: + print("Failed to verify profile page after multiple attempts.") + sender_name = None + + # --- Final Cleanup & Return --- + self.set_clipboard(original_clipboard) # Restore clipboard + # NO cleanup logic (like ESC) here - should be handled by coordinator after this function returns + return sender_name def send_chat_message(self, reply_text: str) -> bool: """Paste text into chat input and send it.""" @@ -558,20 +628,183 @@ class InteractionModule: # ============================================================================== # Position Removal Logic # ============================================================================== -def remove_user_position(detector: DetectionModule, interactor: InteractionModule, trigger_bubble_region: Tuple[int, int, int, int]) -> bool: +def remove_user_position(detector: DetectionModule, + interactor: InteractionModule, + trigger_bubble_region: Tuple[int, int, int, int], # Original region, might be outdated + bubble_snapshot: Any, # PIL Image object for re-location + search_area: Optional[Tuple[int, int, int, int]]) -> bool: # Area to search snapshot in """ Performs the sequence of UI actions to remove a user's position based on the triggering chat bubble. + Includes re-location using the provided snapshot before proceeding. Returns True if successful, False otherwise. """ - print(f"\n--- Starting Position Removal Process (Trigger Bubble Region: {trigger_bubble_region}) ---") - bubble_x, bubble_y, bubble_w, bubble_h = trigger_bubble_region # This is the BBOX, y is top + print(f"\n--- Starting Position Removal Process (Initial Trigger Region: {trigger_bubble_region}) ---") - # 1. Find the closest position icon above the bubble - search_region_y_end = bubble_y - search_region_y_start = max(0, bubble_y - 55) # Search 55 pixels above - search_region_x_start = max(0, bubble_x - 100) # Search wider horizontally + # --- Re-locate Bubble First --- + print("Attempting to re-locate bubble using snapshot before removing position...") + # If bubble_snapshot is None, try to create one from the trigger_bubble_region + if bubble_snapshot is None: + print("Bubble snapshot is missing. Attempting to create a new snapshot from the trigger region...") + try: + if trigger_bubble_region and len(trigger_bubble_region) == 4: + bubble_region_tuple = (int(trigger_bubble_region[0]), int(trigger_bubble_region[1]), + int(trigger_bubble_region[2]), int(trigger_bubble_region[3])) + + if bubble_region_tuple[2] <= 0 or bubble_region_tuple[3] <= 0: + print(f"Warning: Invalid bubble region {bubble_region_tuple} for taking new snapshot.") + return False + + print(f"Taking new screenshot of region: {bubble_region_tuple}") + bubble_snapshot = pyautogui.screenshot(region=bubble_region_tuple) + if bubble_snapshot: + print("Successfully created new bubble snapshot.") + else: + print("Failed to create new bubble snapshot.") + return False + else: + print("Invalid trigger_bubble_region format, cannot create snapshot.") + return False + except Exception as e: + print(f"Error creating new bubble snapshot: {e}") + return False + if search_area is None: + print("Warning: Search area for snapshot is missing. Creating a default search area.") + # Create a default search area centered around the original trigger region + # This creates a search area that's twice the size of the original bubble + if trigger_bubble_region and len(trigger_bubble_region) == 4: + x, y, width, height = trigger_bubble_region + # Expand by 100% in each direction + search_x = max(0, x - width//2) + search_y = max(0, y - height//2) + search_width = width * 2 + search_height = height * 2 + search_area = (search_x, search_y, search_width, search_height) + print(f"Created default search area based on bubble region: {search_area}") + else: + # If no valid trigger_bubble_region, default to full screen search + search_area = None # Set search_area to None for full screen search + print(f"Using full screen search as fallback.") + + # Try to locate the bubble with decreasing confidence levels if needed + new_bubble_box = None + + # Determine the region to search: use provided search_area or None for full screen + region_to_search = search_area + print(f"Attempting bubble location. Search Region: {'Full Screen' if region_to_search is None else region_to_search}") + + # First attempt with standard confidence + print(f"First attempt with confidence {BUBBLE_RELOCATE_CONFIDENCE}...") + try: + new_bubble_box = pyautogui.locateOnScreen(bubble_snapshot, + region=region_to_search, + confidence=BUBBLE_RELOCATE_CONFIDENCE) + except Exception as e: + print(f"Exception during initial bubble location attempt: {e}") + + # Second attempt with fallback confidence if first failed + if not new_bubble_box: + print(f"First attempt failed. Trying with lower confidence {BUBBLE_RELOCATE_FALLBACK_CONFIDENCE}...") + try: + # Try with a lower confidence threshold + new_bubble_box = pyautogui.locateOnScreen(bubble_snapshot, + region=region_to_search, + confidence=BUBBLE_RELOCATE_FALLBACK_CONFIDENCE) + except Exception as e: + print(f"Exception during fallback bubble location attempt: {e}") + + # Third attempt with even lower confidence as last resort + if not new_bubble_box: + print("Second attempt failed. Trying with even lower confidence 0.4...") + try: + # Last resort with very low confidence + new_bubble_box = pyautogui.locateOnScreen(bubble_snapshot, + region=region_to_search, + confidence=0.4) + except Exception as e: + print(f"Exception during last resort bubble location attempt: {e}") + + # If we still can't find the bubble using snapshot, try re-detecting bubbles + if not new_bubble_box: + print("Snapshot location failed. Attempting secondary fallback: Re-detecting bubbles...") + try: + # Helper function to calculate distance - define it here or move globally if used elsewhere + def calculate_distance(p1, p2): + return ((p1[0] - p2[0])**2 + (p1[1] - p2[1])**2)**0.5 + + current_bubbles_info = detector.find_dialogue_bubbles() + non_bot_bubbles = [b for b in current_bubbles_info if not b.get('is_bot')] + + if non_bot_bubbles and trigger_bubble_region and len(trigger_bubble_region) == 4: + original_tl = (trigger_bubble_region[0], trigger_bubble_region[1]) + closest_bubble = None + min_distance = float('inf') + MAX_ALLOWED_DISTANCE = 150 # Example threshold: Don't match bubbles too far away + + for bubble_info in non_bot_bubbles: + bubble_bbox = bubble_info.get('bbox') + if bubble_bbox: + current_tl = (bubble_bbox[0], bubble_bbox[1]) + distance = calculate_distance(original_tl, current_tl) + if distance < min_distance: + min_distance = distance + closest_bubble = bubble_info + + if closest_bubble and min_distance <= MAX_ALLOWED_DISTANCE: + print(f"Found a close bubble via re-detection (Distance: {min_distance:.2f}). Using its bbox.") + bbox = closest_bubble['bbox'] + # Create a dummy box using PyAutoGUI's Box class or a similar structure + from collections import namedtuple + Box = namedtuple('Box', ['left', 'top', 'width', 'height']) + new_bubble_box = Box(left=bbox[0], top=bbox[1], width=bbox[2]-bbox[0], height=bbox[3]-bbox[1]) + print(f"Created fallback bubble box from re-detected bubble: {new_bubble_box}") + else: + print(f"Re-detection fallback failed: No close bubble found (Min distance: {min_distance:.2f} > Threshold: {MAX_ALLOWED_DISTANCE}).") + else: + print("Re-detection fallback failed: No non-bot bubbles found or invalid trigger region.") + + except Exception as redetect_err: + print(f"Error during bubble re-detection fallback: {redetect_err}") + + + # Final fallback: If STILL no bubble box, use original trigger region + if not new_bubble_box: + print("All location attempts failed (snapshot & re-detection). Using original trigger region as last resort.") + if trigger_bubble_region and len(trigger_bubble_region) == 4: + # Create a mock bubble_box from the original region + x, y, width, height = trigger_bubble_region + print(f"Using original trigger region as fallback: {trigger_bubble_region}") + + # Create a dummy box using PyAutoGUI's Box class or a similar structure + from collections import namedtuple + Box = namedtuple('Box', ['left', 'top', 'width', 'height']) + new_bubble_box = Box(left=x, top=y, width=width, height=height) + print("Created fallback bubble box from original coordinates.") + else: + print("Error: No original trigger region available for fallback. Aborting position removal.") + return False + + # Use the NEW coordinates for all subsequent calculations + bubble_x, bubble_y = new_bubble_box.left, new_bubble_box.top + bubble_w, bubble_h = new_bubble_box.width, new_bubble_box.height + print(f"Successfully re-located bubble at: ({bubble_x}, {bubble_y}, {bubble_w}, {bubble_h})") + # --- End Re-location --- + + + # 1. Find the closest position icon above the *re-located* bubble + search_height_pixels = 50 # Search exactly 50 pixels above as requested + search_region_y_end = bubble_y # Use re-located Y + search_region_y_start = max(0, bubble_y - search_height_pixels) # Search 50 pixels above + search_region_x_start = max(0, bubble_x - 100) # Keep horizontal search wide search_region_x_end = bubble_x + bubble_w + 100 - search_region = (search_region_x_start, search_region_y_start, search_region_x_end - search_region_x_start, search_region_y_end - search_region_y_start) + search_region_width = search_region_x_end - search_region_x_start + search_region_height = search_region_y_end - search_region_y_start + + # Ensure region has positive width and height + if search_region_width <= 0 or search_region_height <= 0: + print(f"Error: Invalid search region calculated for position icons: width={search_region_width}, height={search_region_height}") + return False + + search_region = (search_region_x_start, search_region_y_start, search_region_width, search_region_height) print(f"Searching for position icons in region: {search_region}") position_templates = { @@ -579,9 +812,10 @@ def remove_user_position(detector: DetectionModule, interactor: InteractionModul 'SECURITY': POS_SEC_IMG, 'STRATEGY': POS_STR_IMG } found_positions = [] + position_icon_confidence = 0.8 # Slightly increased confidence (was 0.75) for name, path in position_templates.items(): # Use unique keys for detector templates - locations = detector._find_template(name.lower() + '_pos', confidence=0.75, region=search_region) + locations = detector._find_template(name.lower() + '_pos', confidence=position_icon_confidence, region=search_region) for loc in locations: found_positions.append({'name': name, 'coords': loc, 'path': path}) @@ -598,12 +832,12 @@ def remove_user_position(detector: DetectionModule, interactor: InteractionModul target_position_name = closest_position['name'] print(f"Found pending position: |{target_position_name}| at {closest_position['coords']}") - # 2. Click user avatar (offset from bubble top-left) - # IMPORTANT: Use the bubble_y (top of the bbox) for the click Y coordinate. - # The AVATAR_OFFSET_X handles the horizontal positioning relative to the bubble's left edge (bubble_x). - avatar_click_x = bubble_x + AVATAR_OFFSET_X # Use constant offset - avatar_click_y = bubble_y # Use the top Y coordinate of the bubble's bounding box - print(f"Clicking avatar at estimated position: ({avatar_click_x}, {avatar_click_y}) based on bubble top-left ({bubble_x}, {bubble_y})") + # 2. Click user avatar (offset from *re-located* bubble top-left) + # --- MODIFIED: Use specific offsets for remove_position command as requested --- + avatar_click_x = bubble_x + AVATAR_OFFSET_X_REPLY # Use -45 offset + avatar_click_y = bubble_y + AVATAR_OFFSET_Y_REPLY # Use +10 offset + print(f"Clicking avatar for position removal at calculated position: ({avatar_click_x}, {avatar_click_y}) using offsets ({AVATAR_OFFSET_X_REPLY}, {AVATAR_OFFSET_Y_REPLY}) from re-located bubble top-left ({bubble_x}, {bubble_y})") + # --- END MODIFICATION --- interactor.click_at(avatar_click_x, avatar_click_y) time.sleep(0.15) # Wait for profile page @@ -693,7 +927,7 @@ def remove_user_position(detector: DetectionModule, interactor: InteractionModul if close_locs: interactor.click_at(close_locs[0][0], close_locs[0][1]) print("Clicked Close button (returning to Capitol).") - time.sleep(0.15) + time.sleep(0.1) else: print("Warning: Close button not found after confirm, attempting back arrow anyway.") @@ -702,7 +936,7 @@ def remove_user_position(detector: DetectionModule, interactor: InteractionModul if back_arrow_locs: interactor.click_at(back_arrow_locs[0][0], back_arrow_locs[0][1]) print("Clicked Back Arrow (returning to Profile).") - time.sleep(0.15) + time.sleep(0.1) else: print("Warning: Back arrow not found on Capitol page, attempting ESC cleanup.") @@ -800,7 +1034,8 @@ def run_ui_monitoring_loop(trigger_queue: queue.Queue, command_queue: queue.Queu 'page_dev': PAGE_DEV_IMG, 'page_int': PAGE_INT_IMG, 'page_sci': PAGE_SCI_IMG, 'page_sec': PAGE_SEC_IMG, 'page_str': PAGE_STR_IMG, 'dismiss_button': DISMISS_BUTTON_IMG, 'confirm_button': CONFIRM_BUTTON_IMG, - 'close_button': CLOSE_BUTTON_IMG, 'back_arrow': BACK_ARROW_IMG + 'close_button': CLOSE_BUTTON_IMG, 'back_arrow': BACK_ARROW_IMG, + 'reply_button': REPLY_BUTTON_IMG # Added reply button template key } # Use default confidence/region settings from constants detector = DetectionModule(templates, confidence=CONFIDENCE_THRESHOLD, state_confidence=STATE_CONFIDENCE_THRESHOLD, region=SCREENSHOT_REGION) @@ -813,7 +1048,70 @@ def run_ui_monitoring_loop(trigger_queue: queue.Queue, command_queue: queue.Queu screenshot_counter = 0 # Initialize counter for debug screenshots while True: - # --- Check for Main Screen Navigation First --- + # --- Process ALL Pending Commands First --- + commands_processed_this_cycle = False + try: + while True: # Loop to drain the queue + command_data = command_queue.get_nowait() # Check for commands without blocking + commands_processed_this_cycle = True + action = command_data.get('action') + + if action == 'send_reply': + text_to_send = command_data.get('text') + if not text_to_send: + print("UI Thread: Received send_reply command with no text.") + continue # Process next command in queue + print(f"UI Thread: Processing command to send reply: '{text_to_send[:50]}...'") + interactor.send_chat_message(text_to_send) + + elif action == 'remove_position': + # region = command_data.get('trigger_bubble_region') # This is the old region, keep for reference? + snapshot = command_data.get('bubble_snapshot') + area = command_data.get('search_area') + # Pass all necessary data to the function, including the original region if needed for context + # but the function should primarily use the snapshot for re-location. + original_region = command_data.get('trigger_bubble_region') + if snapshot: # Check for snapshot presence + print(f"UI Thread: Processing command to remove position (Snapshot provided: {'Yes' if snapshot else 'No'})") + success = remove_user_position(detector, interactor, original_region, snapshot, area) + print(f"UI Thread: Position removal attempt finished. Success: {success}") + else: + print("UI Thread: Received remove_position command without necessary snapshot data.") + + + elif action == 'pause': + if not monitoring_paused_flag[0]: # Avoid redundant prints if already paused + print("UI Thread: Processing pause command. Pausing monitoring.") + monitoring_paused_flag[0] = True + # No continue needed here, let it finish draining queue + + elif action == 'resume': + if monitoring_paused_flag[0]: # Avoid redundant prints if already running + print("UI Thread: Processing resume command. Resuming monitoring.") + monitoring_paused_flag[0] = False + # No continue needed here + + else: + print(f"UI Thread: Received unknown command: {action}") + + except queue.Empty: + # No more commands in the queue for this cycle + if commands_processed_this_cycle: + print("UI Thread: Finished processing commands for this cycle.") + pass + except Exception as cmd_err: + print(f"UI Thread: Error processing command queue: {cmd_err}") + # Consider if pausing is needed on error, maybe not + + # --- Now, Check Pause State --- + if monitoring_paused_flag[0]: + # If paused, sleep and skip UI monitoring part + time.sleep(0.1) # Sleep briefly while paused + continue # Go back to check commands again + + # --- If not paused, proceed with UI Monitoring --- + + # --- Check for Main Screen Navigation --- try: base_locs = detector._find_template('base_screen', confidence=0.8) map_locs = detector._find_template('world_map_screen', confidence=0.8) @@ -823,7 +1121,7 @@ def run_ui_monitoring_loop(trigger_queue: queue.Queue, command_queue: queue.Queu # IMPORTANT: Ensure these coordinates are correct for the target window/resolution target_x, target_y = 600, 1300 interactor.click_at(target_x, target_y) - time.sleep(0.2) # Short delay after click + time.sleep(0.1) # Short delay after click print("UI Thread: Clicked to return to chat. Re-checking screen state...") continue # Skip the rest of the loop and re-evaluate except Exception as nav_err: @@ -831,34 +1129,52 @@ def run_ui_monitoring_loop(trigger_queue: queue.Queue, command_queue: queue.Queu # Decide if you want to continue or pause after error # --- Process Commands Second (Non-blocking) --- - try: - command_data = command_queue.get_nowait() # Check for commands without blocking - action = command_data.get('action') - if action == 'send_reply': - text_to_send = command_data.get('text') - if text_to_send: - print(f"UI Thread: Received command to send reply: '{text_to_send[:50]}...'") - interactor.send_chat_message(text_to_send) - else: - print("UI Thread: Received send_reply command with no text.") - elif action == 'remove_position': # <--- Handle new command - region = command_data.get('trigger_bubble_region') - if region: - print(f"UI Thread: Received command to remove position triggered by bubble region: {region}") - # Call the new UI function - success = remove_user_position(detector, interactor, region) # Call synchronous function - print(f"UI Thread: Position removal attempt finished. Success: {success}") - # Note: No need to send result back unless main thread needs confirmation - else: - print("UI Thread: Received remove_position command without trigger_bubble_region.") - else: - print(f"UI Thread: Received unknown command: {action}") - except queue.Empty: - pass # No command waiting, continue with monitoring - except Exception as cmd_err: - print(f"UI Thread: Error processing command queue: {cmd_err}") + # This block seems redundant now as commands are processed at the start of the loop. + # Keeping it commented out for now, can be removed later if confirmed unnecessary. + # try: + # command_data = command_queue.get_nowait() # Check for commands without blocking + # action = command_data.get('action') + # if action == 'send_reply': + # text_to_send = command_data.get('text') + # # reply_context_activated = command_data.get('reply_context_activated', False) # Check if reply context was set + # + # if not text_to_send: + # print("UI Thread: Received send_reply command with no text.") + # continue # Skip if no text + # + # print(f"UI Thread: Received command to send reply: '{text_to_send[:50]}...'") + # # The reply context (clicking bubble + reply button) is now handled *before* putting into queue. + # # So, we just need to send the message directly here. + # # The input field should already be focused and potentially have @Username prefix if reply context was activated. + # interactor.send_chat_message(text_to_send) + # + # elif action == 'remove_position': # <--- Handle new command + # region = command_data.get('trigger_bubble_region') + # if region: + # print(f"UI Thread: Received command to remove position triggered by bubble region: {region}") + # # Call the new UI function + # success = remove_user_position(detector, interactor, region) # Call synchronous function + # print(f"UI Thread: Position removal attempt finished. Success: {success}") + # # Note: No need to send result back unless main thread needs confirmation + # else: + # print("UI Thread: Received remove_position command without trigger_bubble_region.") + # elif action == 'pause': # <--- Handle pause command + # print("UI Thread: Received pause command. Pausing monitoring.") + # monitoring_paused_flag[0] = True + # continue # Immediately pause after receiving command + # elif action == 'resume': # <--- Handle resume command (might be redundant if checked above, but safe) + # print("UI Thread: Received resume command. Resuming monitoring.") + # monitoring_paused_flag[0] = False + # else: + # print(f"UI Thread: Received unknown command: {action}") + # except queue.Empty: + # pass # No command waiting, continue with monitoring + # except Exception as cmd_err: + # print(f"UI Thread: Error processing command queue: {cmd_err}") + # # This block is now part of the command processing loop above + # pass - # --- Verify Chat Room State Before Bubble Detection --- + # --- Verify Chat Room State Before Bubble Detection (Only if NOT paused) --- try: # Use a slightly lower confidence maybe, or state_confidence chat_room_locs = detector._find_template('chat_room', confidence=detector.state_confidence) @@ -905,31 +1221,41 @@ def run_ui_monitoring_loop(trigger_queue: queue.Queue, command_queue: queue.Queu if keyword_coords: print(f"\n!!! Keyword detected in bubble {target_bbox} !!!") - # --- Debug Screenshot Logic --- + # --- Variables needed later --- + bubble_snapshot = None # Initialize snapshot variable + search_area = SCREENSHOT_REGION # Define search area early + if search_area is None: + print("Warning: SCREENSHOT_REGION not defined, searching full screen for bubble snapshot.") + # Consider adding a default chat region if SCREENSHOT_REGION is often None + + # --- Take Snapshot for Re-location (and potentially save it) --- try: - screenshot_index = (screenshot_counter % MAX_DEBUG_SCREENSHOTS) + 1 - screenshot_filename = f"debug_bubble_{screenshot_index}.png" - screenshot_path = os.path.join(DEBUG_SCREENSHOT_DIR, screenshot_filename) - # --- Enhanced Logging --- - print(f"Attempting to save debug screenshot to: {screenshot_path}") - print(f"Screenshot Region: {bubble_region}") - # Check if region is valid before attempting screenshot - if bubble_region and len(bubble_region) == 4 and bubble_region[2] > 0 and bubble_region[3] > 0: - # Convert numpy types to standard Python int for pyautogui - region_int = (int(bubble_region[0]), int(bubble_region[1]), int(bubble_region[2]), int(bubble_region[3])) - pyautogui.screenshot(region=region_int, imageFilename=screenshot_path) - print(f"Successfully saved debug screenshot: {screenshot_path}") + bubble_region_tuple = (int(bubble_region[0]), int(bubble_region[1]), int(bubble_region[2]), int(bubble_region[3])) + if bubble_region_tuple[2] <= 0 or bubble_region_tuple[3] <= 0: + print(f"Warning: Invalid bubble region {bubble_region_tuple} for snapshot. Skipping trigger.") + continue + bubble_snapshot = pyautogui.screenshot(region=bubble_region_tuple) + if bubble_snapshot is None: + print("Warning: Failed to capture bubble snapshot. Skipping trigger.") + continue + + # --- Save Snapshot for Debugging (Replaces old debug screenshot logic) --- + try: + screenshot_index = (screenshot_counter % MAX_DEBUG_SCREENSHOTS) + 1 + # Use a more descriptive filename + screenshot_filename = f"debug_relocation_snapshot_{screenshot_index}.png" + screenshot_path = os.path.join(DEBUG_SCREENSHOT_DIR, screenshot_filename) + print(f"Attempting to save bubble snapshot used for re-location to: {screenshot_path}") + bubble_snapshot.save(screenshot_path) # Save the PIL image object + print(f"Successfully saved bubble snapshot: {screenshot_path}") screenshot_counter += 1 - else: - print(f"Error: Invalid screenshot region {bubble_region}. Skipping screenshot.") - # --- End Enhanced Logging --- - except Exception as ss_err: - # --- Enhanced Error Logging --- - print(f"Error taking debug screenshot for region {bubble_region} to path {screenshot_path}: {repr(ss_err)}") - import traceback - traceback.print_exc() # Print full traceback for detailed debugging - # --- End Enhanced Error Logging --- - # --- End Debug Screenshot Logic --- + except Exception as save_err: + print(f"Error saving bubble snapshot to {screenshot_path}: {repr(save_err)}") + # Continue even if saving fails + + except Exception as snapshot_err: + print(f"Error taking initial bubble snapshot: {repr(snapshot_err)}") + continue # Skip trigger if snapshot fails # 4. Interact: Get Bubble Text bubble_text = interactor.copy_text_at(keyword_coords) @@ -949,12 +1275,100 @@ def run_ui_monitoring_loop(trigger_queue: queue.Queue, command_queue: queue.Queu last_processed_bubble_info = target_bubble_info recent_texts.append(bubble_text) - # 5. Interact: Get Sender Name - # *** Use the precise TL coordinates for avatar calculation *** - avatar_coords = detector.calculate_avatar_coords(target_bubble_info['tl_coords']) - sender_name = interactor.retrieve_sender_name_interaction(avatar_coords) + # 5. Interact: Get Sender Name (with Bubble Re-location) + sender_name = None + try: + # --- Bubble Re-location Logic with Fallback Mechanism --- + print("Attempting to re-locate bubble before getting sender name...") + if bubble_snapshot is None: # Should not happen if we reached here, but check anyway + print("Error: Bubble snapshot missing for re-location. Skipping.") + continue + + # First attempt with standard confidence + print(f"First attempt with confidence {BUBBLE_RELOCATE_CONFIDENCE}...") + new_bubble_box = None + try: + new_bubble_box = pyautogui.locateOnScreen(bubble_snapshot, + region=search_area, + confidence=BUBBLE_RELOCATE_CONFIDENCE) + except Exception as e: + print(f"Exception during initial bubble location attempt: {e}") + + # Second attempt with fallback confidence if first failed + if not new_bubble_box: + print(f"First attempt failed. Trying with lower confidence {BUBBLE_RELOCATE_FALLBACK_CONFIDENCE}...") + try: + # Try with a lower confidence threshold + new_bubble_box = pyautogui.locateOnScreen(bubble_snapshot, + region=search_area, + confidence=BUBBLE_RELOCATE_FALLBACK_CONFIDENCE) + except Exception as e: + print(f"Exception during fallback bubble location attempt: {e}") + + # Third attempt with even lower confidence as last resort + if not new_bubble_box: + print("Second attempt failed. Trying with even lower confidence 0.4...") + try: + # Last resort with very low confidence + new_bubble_box = pyautogui.locateOnScreen(bubble_snapshot, + region=search_area, + confidence=0.4) + except Exception as e: + print(f"Exception during last resort bubble location attempt: {e}") + + if new_bubble_box: + new_tl_x, new_tl_y = new_bubble_box.left, new_bubble_box.top + print(f"Successfully re-located bubble snapshot at: ({new_tl_x}, {new_tl_y})") + # Calculate avatar coords based on the *new* top-left and the *reply* offsets + new_avatar_coords = (new_tl_x + AVATAR_OFFSET_X_REPLY, new_tl_y + AVATAR_OFFSET_Y_REPLY) + print(f"Calculated new avatar coordinates for reply context: {new_avatar_coords}") + # Proceed to get sender name using the new coordinates, passing snapshot info for retries + sender_name = interactor.retrieve_sender_name_interaction( + initial_avatar_coords=new_avatar_coords, + bubble_snapshot=bubble_snapshot, + search_area=search_area + ) + else: + print("Warning: Failed to re-locate bubble snapshot on screen after multiple attempts with decreasing confidence thresholds.") + print("Trying direct approach with original bubble coordinates...") + + # Fallback to original coordinates based on the target_bubble_info + original_tl_coords = target_bubble_info.get('tl_coords') + if original_tl_coords: + fallback_avatar_coords = (original_tl_coords[0] + AVATAR_OFFSET_X_REPLY, + original_tl_coords[1] + AVATAR_OFFSET_Y_REPLY) + print(f"Using fallback avatar coordinates from original detection: {fallback_avatar_coords}") + + # Try with direct coordinates + sender_name = interactor.retrieve_sender_name_interaction( + initial_avatar_coords=fallback_avatar_coords, + bubble_snapshot=bubble_snapshot, + search_area=search_area + ) + + if not sender_name: + print("Direct approach failed. Skipping this trigger.") + last_processed_bubble_info = target_bubble_info # Mark as processed + perform_state_cleanup(detector, interactor) # Cleanup + continue + else: + print("No original coordinates available. Skipping sender name retrieval.") + # No need to continue if we can't find the bubble again + last_processed_bubble_info = target_bubble_info # Mark as processed to avoid re-triggering immediately + perform_state_cleanup(detector, interactor) # Attempt cleanup as state might be inconsistent + continue + # --- End Bubble Re-location Logic --- + + except Exception as reloc_err: + print(f"Error during bubble re-location or subsequent interaction: {reloc_err}") + import traceback + traceback.print_exc() + # Attempt cleanup after error during this critical phase + perform_state_cleanup(detector, interactor) + continue # Skip further processing for this trigger # 6. Perform Cleanup (Crucial after potentially leaving chat screen) + # Moved the check for sender_name *after* potential re-location attempt cleanup_successful = perform_state_cleanup(detector, interactor) if not cleanup_successful: print("Error: Failed to return to chat screen after getting name. Aborting trigger.") @@ -964,21 +1378,71 @@ def run_ui_monitoring_loop(trigger_queue: queue.Queue, command_queue: queue.Queu print("Error: Could not get sender name, aborting processing.") continue # Already cleaned up, just skip + # --- Attempt to activate reply context BEFORE putting in queue --- + reply_context_activated = False + try: + print("Attempting to activate reply context...") + # Re-locate the bubble *again* to click its center for reply + if bubble_snapshot is None: + print("Warning: Bubble snapshot missing for reply context activation. Skipping.") + final_bubble_box_for_reply = None # Ensure it's None + else: + print(f"Attempting final re-location for reply context using search_area: {search_area}") + final_bubble_box_for_reply = pyautogui.locateOnScreen(bubble_snapshot, region=search_area, confidence=BUBBLE_RELOCATE_CONFIDENCE) + + if final_bubble_box_for_reply: + print(f"Final re-location successful at: {final_bubble_box_for_reply}") + bubble_x_reply, bubble_y_reply = final_bubble_box_for_reply.left, final_bubble_box_for_reply.top + bubble_w_reply, bubble_h_reply = final_bubble_box_for_reply.width, final_bubble_box_for_reply.height + center_x_reply = bubble_x_reply + bubble_w_reply // 2 + center_y_reply = bubble_y_reply + bubble_h_reply // 2 + + print(f"Clicking bubble center for reply at ({center_x_reply}, {center_y_reply})") + interactor.click_at(center_x_reply, center_y_reply) + time.sleep(0.15) # Increased wait time for menu/reply button to appear + + print("Searching for reply button...") + reply_button_locs = detector._find_template('reply_button', confidence=0.8) + if reply_button_locs: + reply_coords = reply_button_locs[0] + print(f"Found reply button at {reply_coords}. Clicking...") + interactor.click_at(reply_coords[0], reply_coords[1]) + time.sleep(0.07) # Wait after click + reply_context_activated = True + print("Reply context activated.") + else: + print(">>> Reply button template ('reply_button') not found after clicking bubble center. <<<") + # Optional: Press ESC to close menu if reply button wasn't found? + # print("Attempting to press ESC to close potential menu.") + # interactor.press_key('esc') + # time.sleep(0.1) + else: + # This log message was already present but is important + print("Warning: Failed to re-locate bubble for activating reply context.") + + except Exception as reply_context_err: + print(f"!!! Error during reply context activation: {reply_context_err} !!!") + # Ensure reply_context_activated remains False + # 7. Send Trigger Info to Main Thread/Async Loop print("\n>>> Putting trigger info in Queue <<<") print(f" Sender: {sender_name}") print(f" Content: {bubble_text[:100]}...") print(f" Bubble Region: {bubble_region}") # Include region derived from bbox + print(f" Reply Context Activated: {reply_context_activated}") # Include the flag try: - # Include bubble_region in the data sent + # Include bubble_region and reply_context_activated flag data_to_send = { 'sender': sender_name, 'text': bubble_text, - 'bubble_region': bubble_region # Use bbox-derived region for general use + 'bubble_region': bubble_region, # Use bbox-derived region for general use + 'reply_context_activated': reply_context_activated, # Send the flag + 'bubble_snapshot': bubble_snapshot, # <-- Add snapshot + 'search_area': search_area # <-- Add search area used for snapshot # 'tl_coords': target_bubble_info['tl_coords'] # Optionally send if needed elsewhere } trigger_queue.put(data_to_send) # Put in the queue for main loop - print("Trigger info (with region) placed in Queue.") + print("Trigger info (with region, reply flag, snapshot, search_area) placed in Queue.") except Exception as q_err: print(f"Error putting data in Queue: {q_err}")