Merge pull request #1 from z060142/Refactoring
Refactoring and modularize existing work logic
26
main.py
@ -24,8 +24,9 @@ all_discovered_mcp_tools: list[dict] = []
|
|||||||
exit_stack = AsyncExitStack()
|
exit_stack = AsyncExitStack()
|
||||||
# Stores loaded persona data (as a string for easy injection into prompt)
|
# Stores loaded persona data (as a string for easy injection into prompt)
|
||||||
wolfhart_persona_details: str | None = None
|
wolfhart_persona_details: str | None = None
|
||||||
# --- Use standard thread-safe queue ---
|
# --- Use standard thread-safe queues ---
|
||||||
trigger_queue: ThreadSafeQueue = ThreadSafeQueue() # Use standard Queue
|
trigger_queue: ThreadSafeQueue = ThreadSafeQueue() # UI Thread -> Main Loop
|
||||||
|
command_queue: ThreadSafeQueue = ThreadSafeQueue() # Main Loop -> UI Thread
|
||||||
# --- End Change ---
|
# --- End Change ---
|
||||||
ui_monitor_task: asyncio.Task | None = None # To track the UI monitor task
|
ui_monitor_task: asyncio.Task | None = None # To track the UI monitor task
|
||||||
|
|
||||||
@ -205,8 +206,9 @@ async def run_main_with_exit_stack():
|
|||||||
# 3. Start UI Monitoring in a separate thread
|
# 3. Start UI Monitoring in a separate thread
|
||||||
print("\n--- Starting UI monitoring thread ---")
|
print("\n--- Starting UI monitoring thread ---")
|
||||||
loop = asyncio.get_running_loop() # Get loop for run_in_executor
|
loop = asyncio.get_running_loop() # Get loop for run_in_executor
|
||||||
|
# Use the new monitoring loop function, passing both queues
|
||||||
monitor_task = loop.create_task(
|
monitor_task = loop.create_task(
|
||||||
asyncio.to_thread(ui_interaction.monitor_chat_for_trigger, trigger_queue),
|
asyncio.to_thread(ui_interaction.run_ui_monitoring_loop, trigger_queue, command_queue), # Pass command_queue
|
||||||
name="ui_monitor"
|
name="ui_monitor"
|
||||||
)
|
)
|
||||||
ui_monitor_task = monitor_task # Store task reference for shutdown
|
ui_monitor_task = monitor_task # Store task reference for shutdown
|
||||||
@ -265,15 +267,16 @@ async def run_main_with_exit_stack():
|
|||||||
if thoughts:
|
if thoughts:
|
||||||
print(f"AI Thoughts: {thoughts[:150]}..." if len(thoughts) > 150 else f"AI Thoughts: {thoughts}")
|
print(f"AI Thoughts: {thoughts[:150]}..." if len(thoughts) > 150 else f"AI Thoughts: {thoughts}")
|
||||||
|
|
||||||
# 只有當有效回應時才發送到遊戲
|
# 只有當有效回應時才發送到遊戲 (via command queue)
|
||||||
if bot_dialogue and valid_response:
|
if bot_dialogue and valid_response:
|
||||||
print("Preparing to send dialogue response via UI...")
|
print("Sending 'send_reply' command to UI thread...")
|
||||||
send_success = await asyncio.to_thread(
|
command_to_send = {'action': 'send_reply', 'text': bot_dialogue}
|
||||||
ui_interaction.paste_and_send_reply,
|
try:
|
||||||
bot_dialogue
|
# Put command into the queue for the UI thread to handle
|
||||||
)
|
await loop.run_in_executor(None, command_queue.put, command_to_send)
|
||||||
if send_success: print("Response sent successfully.")
|
print("Command placed in queue.")
|
||||||
else: print("Error: Failed to send response via UI.")
|
except Exception as q_err:
|
||||||
|
print(f"Error putting command in queue: {q_err}")
|
||||||
else:
|
else:
|
||||||
print("Not sending response: Invalid or empty dialogue content.")
|
print("Not sending response: Invalid or empty dialogue content.")
|
||||||
|
|
||||||
@ -309,4 +312,3 @@ if __name__ == "__main__":
|
|||||||
print(f"Top-level error during asyncio.run execution: {e}")
|
print(f"Top-level error during asyncio.run execution: {e}")
|
||||||
finally:
|
finally:
|
||||||
print("Program exited.")
|
print("Program exited.")
|
||||||
|
|
||||||
|
|||||||
BIN
templates/In_private_room.png
Normal file
|
After Width: | Height: | Size: 4.0 KiB |
BIN
templates/In_world_room.png
Normal file
|
After Width: | Height: | Size: 4.0 KiB |
BIN
templates/Previous_page.png
Normal file
|
After Width: | Height: | Size: 4.6 KiB |
BIN
templates/Private_Label_normal.png
Normal file
|
After Width: | Height: | Size: 4.4 KiB |
BIN
templates/World_Label_normal.png
Normal file
|
After Width: | Height: | Size: 3.3 KiB |
BIN
templates/World_map.png
Normal file
|
After Width: | Height: | Size: 15 KiB |
|
Before Width: | Height: | Size: 5.7 KiB After Width: | Height: | Size: 4.6 KiB |
@ -1,6 +1,5 @@
|
|||||||
# ui_interaction.py
|
# ui_interaction.py
|
||||||
# Handles recognition and interaction logic with the game screen
|
# Refactored to separate Detection and Interaction logic.
|
||||||
# Includes: Bot bubble corner detection, case-sensitive keyword detection, duplicate handling mechanism, state-based ESC cleanup, complete syntax fixes
|
|
||||||
|
|
||||||
import pyautogui
|
import pyautogui
|
||||||
import cv2 # opencv-python
|
import cv2 # opencv-python
|
||||||
@ -13,183 +12,144 @@ import asyncio
|
|||||||
import pygetwindow as gw # Used to check/activate windows
|
import pygetwindow as gw # Used to check/activate windows
|
||||||
import config # Used to read window title
|
import config # Used to read window title
|
||||||
import queue
|
import queue
|
||||||
|
from typing import List, Tuple, Optional, Dict, Any
|
||||||
|
|
||||||
# --- Configuration Section ---
|
# --- Configuration Section ---
|
||||||
# Get script directory to ensure relative paths are correct
|
|
||||||
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||||
TEMPLATE_DIR = os.path.join(SCRIPT_DIR, "templates") # Templates image folder path
|
TEMPLATE_DIR = os.path.join(SCRIPT_DIR, "templates")
|
||||||
os.makedirs(TEMPLATE_DIR, exist_ok=True) # Ensure folder exists
|
os.makedirs(TEMPLATE_DIR, exist_ok=True)
|
||||||
|
|
||||||
# --- Regular Bubble Corner Templates ---
|
# --- Template Paths (Consider moving to config.py or loading dynamically) ---
|
||||||
# Please save screenshots to the templates folder using the following filenames
|
# Bubble Corners
|
||||||
CORNER_TL_IMG = os.path.join(TEMPLATE_DIR, "corner_tl.png") # Regular bubble - Top Left corner
|
CORNER_TL_IMG = os.path.join(TEMPLATE_DIR, "corner_tl.png")
|
||||||
CORNER_TR_IMG = os.path.join(TEMPLATE_DIR, "corner_tr.png") # Regular bubble - Top Right corner
|
CORNER_TR_IMG = os.path.join(TEMPLATE_DIR, "corner_tr.png")
|
||||||
CORNER_BL_IMG = os.path.join(TEMPLATE_DIR, "corner_bl.png") # Regular bubble - Bottom Left corner
|
CORNER_BL_IMG = os.path.join(TEMPLATE_DIR, "corner_bl.png")
|
||||||
CORNER_BR_IMG = os.path.join(TEMPLATE_DIR, "corner_br.png") # Regular bubble - Bottom Right corner
|
CORNER_BR_IMG = os.path.join(TEMPLATE_DIR, "corner_br.png")
|
||||||
|
BOT_CORNER_TL_IMG = os.path.join(TEMPLATE_DIR, "bot_corner_tl.png")
|
||||||
|
BOT_CORNER_TR_IMG = os.path.join(TEMPLATE_DIR, "bot_corner_tr.png")
|
||||||
|
BOT_CORNER_BL_IMG = os.path.join(TEMPLATE_DIR, "bot_corner_bl.png")
|
||||||
|
BOT_CORNER_BR_IMG = os.path.join(TEMPLATE_DIR, "bot_corner_br.png")
|
||||||
|
# Keywords
|
||||||
|
KEYWORD_wolf_LOWER_IMG = os.path.join(TEMPLATE_DIR, "keyword_wolf_lower.png")
|
||||||
|
KEYWORD_Wolf_UPPER_IMG = os.path.join(TEMPLATE_DIR, "keyword_wolf_upper.png")
|
||||||
|
# UI Elements
|
||||||
|
COPY_MENU_ITEM_IMG = os.path.join(TEMPLATE_DIR, "copy_menu_item.png")
|
||||||
|
PROFILE_OPTION_IMG = os.path.join(TEMPLATE_DIR, "profile_option.png")
|
||||||
|
COPY_NAME_BUTTON_IMG = os.path.join(TEMPLATE_DIR, "copy_name_button.png")
|
||||||
|
SEND_BUTTON_IMG = os.path.join(TEMPLATE_DIR, "send_button.png")
|
||||||
|
CHAT_INPUT_IMG = os.path.join(TEMPLATE_DIR, "chat_input.png")
|
||||||
|
# State Detection
|
||||||
|
PROFILE_NAME_PAGE_IMG = os.path.join(TEMPLATE_DIR, "Profile_Name_page.png")
|
||||||
|
PROFILE_PAGE_IMG = os.path.join(TEMPLATE_DIR, "Profile_page.png")
|
||||||
|
CHAT_ROOM_IMG = os.path.join(TEMPLATE_DIR, "chat_room.png")
|
||||||
|
# Add World/Private chat identifiers later
|
||||||
|
WORLD_CHAT_IMG = os.path.join(TEMPLATE_DIR, "World_Label_normal.png") # Example
|
||||||
|
PRIVATE_CHAT_IMG = os.path.join(TEMPLATE_DIR, "Private_Label_normal.png") # Example
|
||||||
|
|
||||||
# --- Bot Bubble Corner Templates (need to be provided!) ---
|
# --- Operation Parameters (Consider moving to config.py) ---
|
||||||
# Please save screenshots to the templates folder using the following filenames
|
CHAT_INPUT_REGION = None # Example: (100, 800, 500, 50)
|
||||||
BOT_CORNER_TL_IMG = os.path.join(TEMPLATE_DIR, "bot_corner_tl.png") # Bot bubble - Top Left corner
|
CHAT_INPUT_CENTER_X = 400
|
||||||
BOT_CORNER_TR_IMG = os.path.join(TEMPLATE_DIR, "bot_corner_tr.png") # Bot bubble - Top Right corner
|
CHAT_INPUT_CENTER_Y = 1280
|
||||||
BOT_CORNER_BL_IMG = os.path.join(TEMPLATE_DIR, "bot_corner_bl.png") # Bot bubble - Bottom Left corner
|
SCREENSHOT_REGION = None
|
||||||
BOT_CORNER_BR_IMG = os.path.join(TEMPLATE_DIR, "bot_corner_br.png") # Bot bubble - Bottom Right corner
|
CONFIDENCE_THRESHOLD = 0.8
|
||||||
|
STATE_CONFIDENCE_THRESHOLD = 0.7
|
||||||
|
AVATAR_OFFSET_X = -50
|
||||||
|
BBOX_SIMILARITY_TOLERANCE = 10
|
||||||
|
RECENT_TEXT_HISTORY_MAXLEN = 5 # This state likely belongs in the coordinator
|
||||||
|
|
||||||
# --- Keyword Templates (case-sensitive) ---
|
# --- Helper Function (Module Level) ---
|
||||||
# Please save screenshots to the templates folder using the following filenames
|
def are_bboxes_similar(bbox1: Optional[Tuple[int, int, int, int]],
|
||||||
KEYWORD_wolf_LOWER_IMG = os.path.join(TEMPLATE_DIR, "keyword_wolf_lower.png") # Lowercase "wolf"
|
bbox2: Optional[Tuple[int, int, int, int]],
|
||||||
KEYWORD_Wolf_UPPER_IMG = os.path.join(TEMPLATE_DIR, "keyword_wolf_upper.png") # Uppercase "Wolf"
|
tolerance: int = BBOX_SIMILARITY_TOLERANCE) -> bool:
|
||||||
|
"""Check if two bounding boxes' top-left corners are close."""
|
||||||
|
if bbox1 is None or bbox2 is None:
|
||||||
|
return False
|
||||||
|
return abs(bbox1[0] - bbox2[0]) <= tolerance and abs(bbox1[1] - bbox2[1]) <= tolerance
|
||||||
|
|
||||||
# --- UI Element Templates ---
|
# ==============================================================================
|
||||||
# Please save screenshots to the templates folder using the following filenames
|
# Detection Module
|
||||||
COPY_MENU_ITEM_IMG = os.path.join(TEMPLATE_DIR, "copy_menu_item.png") # "Copy" option in the menu
|
# ==============================================================================
|
||||||
PROFILE_OPTION_IMG = os.path.join(TEMPLATE_DIR, "profile_option.png") # Option in the profile card that opens user details
|
class DetectionModule:
|
||||||
COPY_NAME_BUTTON_IMG = os.path.join(TEMPLATE_DIR, "copy_name_button.png") # "Copy Name" button in user details
|
"""Handles finding elements and states on the screen using image recognition."""
|
||||||
SEND_BUTTON_IMG = os.path.join(TEMPLATE_DIR, "send_button.png") # "Send" button for the chat input box
|
|
||||||
CHAT_INPUT_IMG = os.path.join(TEMPLATE_DIR, "chat_input.png") # (Optional) Template image for the chat input box
|
|
||||||
|
|
||||||
# --- Status Detection Templates ---
|
def __init__(self, templates: Dict[str, str], confidence: float = CONFIDENCE_THRESHOLD, state_confidence: float = STATE_CONFIDENCE_THRESHOLD, region: Optional[Tuple[int, int, int, int]] = SCREENSHOT_REGION):
|
||||||
# Please save screenshots to the templates folder using the following filenames
|
self.templates = templates
|
||||||
PROFILE_NAME_PAGE_IMG = os.path.join(TEMPLATE_DIR, "Profile_Name_page.png") # User details page identifier
|
self.confidence = confidence
|
||||||
PROFILE_PAGE_IMG = os.path.join(TEMPLATE_DIR, "Profile_page.png") # Profile card page identifier
|
self.state_confidence = state_confidence
|
||||||
CHAT_ROOM_IMG = os.path.join(TEMPLATE_DIR, "chat_room.png") # Chat room interface identifier
|
self.region = region
|
||||||
|
self._warned_paths = set()
|
||||||
|
print("DetectionModule initialized.")
|
||||||
|
|
||||||
# --- Operation Parameters (need to be adjusted based on your environment) ---
|
def _find_template(self, template_key: str, confidence: Optional[float] = None, region: Optional[Tuple[int, int, int, int]] = None, grayscale: bool = False) -> List[Tuple[int, int]]:
|
||||||
# Chat input box reference coordinates or region (needed if not using image positioning)
|
"""Internal helper to find a template by its key."""
|
||||||
CHAT_INPUT_REGION = None # (100, 800, 500, 50) # Example region (x, y, width, height)
|
template_path = self.templates.get(template_key)
|
||||||
CHAT_INPUT_CENTER_X = 400 # Example X coordinate
|
if not template_path:
|
||||||
CHAT_INPUT_CENTER_Y = 1280 # Example Y coordinate
|
print(f"Error: Template key '{template_key}' not found in provided templates.")
|
||||||
|
|
||||||
# Screenshot and recognition parameters
|
|
||||||
SCREENSHOT_REGION = None # None means full screen, or set to (x, y, width, height) to limit scanning area
|
|
||||||
CONFIDENCE_THRESHOLD = 0.8 # Main image matching confidence threshold (0.0 ~ 1.0), needs adjustment
|
|
||||||
STATE_CONFIDENCE_THRESHOLD = 0.7 # State detection confidence threshold (may need to be lower)
|
|
||||||
AVATAR_OFFSET_X = -50 # Avatar X offset relative to bubble top-left corner (based on your update)
|
|
||||||
|
|
||||||
# Duplicate handling parameters
|
|
||||||
BBOX_SIMILARITY_TOLERANCE = 10 # Pixel tolerance when determining if two bubbles are in similar positions
|
|
||||||
RECENT_TEXT_HISTORY_MAXLEN = 5 # Number of recently processed texts to keep
|
|
||||||
|
|
||||||
# --- Helper Functions ---
|
|
||||||
|
|
||||||
def find_template_on_screen(template_path, region=None, confidence=CONFIDENCE_THRESHOLD, grayscale=False):
|
|
||||||
"""
|
|
||||||
Find a template image in a specified screen region (more robust version).
|
|
||||||
|
|
||||||
Args:
|
|
||||||
template_path (str): Path to the template image.
|
|
||||||
region (tuple, optional): Screenshot region (x, y, width, height). Default is None (full screen).
|
|
||||||
confidence (float, optional): Matching confidence threshold. Default is CONFIDENCE_THRESHOLD.
|
|
||||||
grayscale (bool, optional): Whether to use grayscale for matching. Default is False.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
list: List containing center point coordinates of all found matches [(x1, y1), (x2, y2), ...],
|
|
||||||
or empty list if none found.
|
|
||||||
"""
|
|
||||||
locations = []
|
|
||||||
# Check if template file exists, warn only once when not found
|
|
||||||
if not os.path.exists(template_path):
|
|
||||||
if not hasattr(find_template_on_screen, 'warned_paths'):
|
|
||||||
find_template_on_screen.warned_paths = set()
|
|
||||||
if template_path not in find_template_on_screen.warned_paths:
|
|
||||||
print(f"Error: Template image doesn't exist: {template_path}")
|
|
||||||
find_template_on_screen.warned_paths.add(template_path)
|
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
# Check if template file exists, warn only once
|
||||||
|
if not os.path.exists(template_path):
|
||||||
|
if template_path not in self._warned_paths:
|
||||||
|
print(f"Error: Template image doesn't exist: {template_path}")
|
||||||
|
self._warned_paths.add(template_path)
|
||||||
|
return []
|
||||||
|
|
||||||
|
locations = []
|
||||||
|
current_region = region if region is not None else self.region
|
||||||
|
current_confidence = confidence if confidence is not None else self.confidence
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Use pyautogui to find all matches (requires opencv-python)
|
matches = pyautogui.locateAllOnScreen(template_path, region=current_region, confidence=current_confidence, grayscale=grayscale)
|
||||||
matches = pyautogui.locateAllOnScreen(template_path, region=region, confidence=confidence, grayscale=grayscale)
|
|
||||||
if matches:
|
if matches:
|
||||||
for box in matches:
|
for box in matches:
|
||||||
center_x = box.left + box.width // 2
|
center_x = box.left + box.width // 2
|
||||||
center_y = box.top + box.height // 2
|
center_y = box.top + box.height // 2
|
||||||
locations.append((center_x, center_y))
|
locations.append((center_x, center_y))
|
||||||
# print(f"Found template '{os.path.basename(template_path)}' at {len(locations)} locations.") # Debug
|
# print(f"Found template '{template_key}' at {len(locations)} locations.") # Debug
|
||||||
return locations
|
return locations
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
# Print more detailed error, including template path
|
print(f"Error finding template '{template_key}' ({template_path}): {e}")
|
||||||
print(f"Error finding template '{os.path.basename(template_path)}' ({template_path}): {e}")
|
|
||||||
return []
|
return []
|
||||||
|
|
||||||
def click_at(x, y, button='left', clicks=1, interval=0.1, duration=0.1):
|
def find_elements(self, template_keys: List[str], confidence: Optional[float] = None, region: Optional[Tuple[int, int, int, int]] = None) -> Dict[str, List[Tuple[int, int]]]:
|
||||||
"""Safely click at specific coordinates, with movement time added"""
|
"""Find multiple templates by their keys."""
|
||||||
try:
|
results = {}
|
||||||
x_int, y_int = int(x), int(y) # Ensure coordinates are integers
|
for key in template_keys:
|
||||||
print(f"Moving to and clicking at: ({x_int}, {y_int}), button: {button}, clicks: {clicks}")
|
results[key] = self._find_template(key, confidence=confidence, region=region)
|
||||||
pyautogui.moveTo(x_int, y_int, duration=duration) # Smooth move to target
|
return results
|
||||||
pyautogui.click(button=button, clicks=clicks, interval=interval)
|
|
||||||
time.sleep(0.1) # Brief pause after clicking
|
|
||||||
except Exception as e:
|
|
||||||
print(f"Error clicking at coordinates ({int(x)}, {int(y)}): {e}")
|
|
||||||
|
|
||||||
def get_clipboard_text():
|
def find_dialogue_bubbles(self) -> List[Tuple[Tuple[int, int, int, int], bool]]:
|
||||||
"""Get text from clipboard"""
|
|
||||||
try:
|
|
||||||
return pyperclip.paste()
|
|
||||||
except Exception as e:
|
|
||||||
# pyperclip might fail in certain environments (like headless servers)
|
|
||||||
print(f"Error reading clipboard: {e}")
|
|
||||||
return None
|
|
||||||
|
|
||||||
def set_clipboard_text(text):
|
|
||||||
"""Set clipboard text"""
|
|
||||||
try:
|
|
||||||
pyperclip.copy(text)
|
|
||||||
except Exception as e:
|
|
||||||
print(f"Error writing to clipboard: {e}")
|
|
||||||
|
|
||||||
def are_bboxes_similar(bbox1, bbox2, tolerance=BBOX_SIMILARITY_TOLERANCE):
|
|
||||||
"""Check if two bounding boxes' positions (top-left corner) are very close"""
|
|
||||||
if bbox1 is None or bbox2 is None:
|
|
||||||
return False
|
|
||||||
# Compare top-left coordinates (bbox[0], bbox[1])
|
|
||||||
return abs(bbox1[0] - bbox2[0]) <= tolerance and abs(bbox1[1] - bbox2[1]) <= tolerance
|
|
||||||
|
|
||||||
# --- Main Logic Functions ---
|
|
||||||
|
|
||||||
def find_dialogue_bubbles():
|
|
||||||
"""
|
"""
|
||||||
Scan the screen for regular bubble corners and Bot bubble corners, and try to pair them.
|
Scan screen for regular and bot bubble corners and pair them.
|
||||||
Returns a list containing bounding boxes and whether they are Bot bubbles.
|
Returns list of (bbox, is_bot_flag). Basic matching logic.
|
||||||
!!! The matching logic is very basic and needs significant improvement based on actual needs !!!
|
|
||||||
"""
|
"""
|
||||||
all_bubbles_with_type = [] # Store (bbox, is_bot_flag)
|
all_bubbles_with_type = []
|
||||||
|
|
||||||
# 1. Find all regular corners
|
# Find corners using the internal helper
|
||||||
tl_corners = find_template_on_screen(CORNER_TL_IMG, region=SCREENSHOT_REGION)
|
tl_corners = self._find_template('corner_tl')
|
||||||
br_corners = find_template_on_screen(CORNER_BR_IMG, region=SCREENSHOT_REGION)
|
br_corners = self._find_template('corner_br')
|
||||||
# tr_corners = find_template_on_screen(CORNER_TR_IMG, region=SCREENSHOT_REGION) # Not using TR/BL for now
|
bot_tl_corners = self._find_template('bot_corner_tl')
|
||||||
# bl_corners = find_template_on_screen(CORNER_BL_IMG, region=SCREENSHOT_REGION)
|
bot_br_corners = self._find_template('bot_corner_br')
|
||||||
|
|
||||||
# 2. Find all Bot corners
|
# Match regular bubbles
|
||||||
bot_tl_corners = find_template_on_screen(BOT_CORNER_TL_IMG, region=SCREENSHOT_REGION)
|
processed_tls = set()
|
||||||
bot_br_corners = find_template_on_screen(BOT_CORNER_BR_IMG, region=SCREENSHOT_REGION)
|
|
||||||
# bot_tr_corners = find_template_on_screen(BOT_CORNER_TR_IMG, region=SCREENSHOT_REGION)
|
|
||||||
# bot_bl_corners = find_template_on_screen(BOT_CORNER_BL_IMG, region=SCREENSHOT_REGION)
|
|
||||||
|
|
||||||
# 3. Try to match regular bubbles (using TL and BR)
|
|
||||||
processed_tls = set() # Track already matched TL indices
|
|
||||||
if tl_corners and br_corners:
|
if tl_corners and br_corners:
|
||||||
for i, tl in enumerate(tl_corners):
|
for i, tl in enumerate(tl_corners):
|
||||||
if i in processed_tls: continue
|
if i in processed_tls: continue
|
||||||
potential_br = None
|
potential_br = None
|
||||||
min_dist_sq = float('inf')
|
min_dist_sq = float('inf')
|
||||||
# Find the best BR corresponding to this TL (e.g., closest, or satisfying specific geometric constraints)
|
|
||||||
for j, br in enumerate(br_corners):
|
for j, br in enumerate(br_corners):
|
||||||
# Check if BR is in a reasonable range to the bottom-right of TL
|
if br[0] > tl[0] + 20 and br[1] > tl[1] + 10:
|
||||||
if br[0] > tl[0] + 20 and br[1] > tl[1] + 10: # Assume minimum width/height
|
|
||||||
dist_sq = (br[0] - tl[0])**2 + (br[1] - tl[1])**2
|
dist_sq = (br[0] - tl[0])**2 + (br[1] - tl[1])**2
|
||||||
# Could add more conditions here, e.g., aspect ratio limits
|
if dist_sq < min_dist_sq:
|
||||||
if dist_sq < min_dist_sq: # Simple nearest-match
|
|
||||||
potential_br = br
|
potential_br = br
|
||||||
min_dist_sq = dist_sq
|
min_dist_sq = dist_sq
|
||||||
|
|
||||||
if potential_br:
|
if potential_br:
|
||||||
# Assuming we found matching TL and BR, define bounding box
|
|
||||||
bubble_bbox = (tl[0], tl[1], potential_br[0], potential_br[1])
|
bubble_bbox = (tl[0], tl[1], potential_br[0], potential_br[1])
|
||||||
all_bubbles_with_type.append((bubble_bbox, False)) # Mark as non-Bot
|
all_bubbles_with_type.append((bubble_bbox, False))
|
||||||
processed_tls.add(i) # Mark this TL as used
|
processed_tls.add(i)
|
||||||
|
|
||||||
# 4. Try to match Bot bubbles (using Bot TL and Bot BR)
|
# Match Bot bubbles
|
||||||
processed_bot_tls = set()
|
processed_bot_tls = set()
|
||||||
if bot_tl_corners and bot_br_corners:
|
if bot_tl_corners and bot_br_corners:
|
||||||
for i, tl in enumerate(bot_tl_corners):
|
for i, tl in enumerate(bot_tl_corners):
|
||||||
@ -204,250 +164,253 @@ def find_dialogue_bubbles():
|
|||||||
min_dist_sq = dist_sq
|
min_dist_sq = dist_sq
|
||||||
if potential_br:
|
if potential_br:
|
||||||
bubble_bbox = (tl[0], tl[1], potential_br[0], potential_br[1])
|
bubble_bbox = (tl[0], tl[1], potential_br[0], potential_br[1])
|
||||||
all_bubbles_with_type.append((bubble_bbox, True)) # Mark as Bot
|
all_bubbles_with_type.append((bubble_bbox, True))
|
||||||
processed_bot_tls.add(i)
|
processed_bot_tls.add(i)
|
||||||
|
|
||||||
# print(f"Found {len(all_bubbles_with_type)} potential bubbles.") #reduce printing
|
|
||||||
return all_bubbles_with_type
|
return all_bubbles_with_type
|
||||||
|
|
||||||
|
def find_keyword_in_region(self, region: Tuple[int, int, int, int]) -> Optional[Tuple[int, int]]:
|
||||||
|
"""Look for keywords within a specified region."""
|
||||||
|
if region[2] <= 0 or region[3] <= 0: return None # Invalid region width/height
|
||||||
|
|
||||||
def find_keyword_in_bubble(bubble_bbox):
|
# Try lowercase
|
||||||
"""
|
locations_lower = self._find_template('keyword_wolf_lower', region=region)
|
||||||
Look for the keywords "wolf" or "Wolf" images within the specified bubble area.
|
if locations_lower:
|
||||||
"""
|
print(f"Found keyword (lowercase) in region {region}, position: {locations_lower[0]}")
|
||||||
x_min, y_min, x_max, y_max = bubble_bbox
|
return locations_lower[0]
|
||||||
width = x_max - x_min
|
|
||||||
height = y_max - y_min
|
|
||||||
if width <= 0 or height <= 0:
|
|
||||||
# print(f"Warning: Invalid bubble area {bubble_bbox}") #reduce printing
|
|
||||||
return None
|
|
||||||
search_region = (x_min, y_min, width, height)
|
|
||||||
# print(f"Searching for keywords in region {search_region}...") #reduce printing
|
|
||||||
|
|
||||||
# 1. Try to find lowercase "wolf"
|
# Try uppercase
|
||||||
keyword_locations_lower = find_template_on_screen(KEYWORD_wolf_LOWER_IMG, region=search_region)
|
locations_upper = self._find_template('keyword_wolf_upper', region=region)
|
||||||
if keyword_locations_lower:
|
if locations_upper:
|
||||||
keyword_coords = keyword_locations_lower[0]
|
print(f"Found keyword (uppercase) in region {region}, position: {locations_upper[0]}")
|
||||||
print(f"Found keyword (lowercase) in bubble {bubble_bbox}, position: {keyword_coords}")
|
return locations_upper[0]
|
||||||
return keyword_coords
|
|
||||||
|
|
||||||
# 2. If lowercase not found, try uppercase "Wolf"
|
|
||||||
keyword_locations_upper = find_template_on_screen(KEYWORD_Wolf_UPPER_IMG, region=search_region)
|
|
||||||
if keyword_locations_upper:
|
|
||||||
keyword_coords = keyword_locations_upper[0]
|
|
||||||
print(f"Found keyword (uppercase) in bubble {bubble_bbox}, position: {keyword_coords}")
|
|
||||||
return keyword_coords
|
|
||||||
|
|
||||||
# If neither found
|
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def find_avatar_for_bubble(bubble_bbox):
|
def calculate_avatar_coords(self, bubble_bbox: Tuple[int, int, int, int], offset_x: int = AVATAR_OFFSET_X) -> Tuple[int, int]:
|
||||||
"""Calculate avatar frame position based on bubble's top-left coordinates."""
|
"""Calculate avatar coordinates based on bubble top-left."""
|
||||||
tl_x, tl_y = bubble_bbox[0], bubble_bbox[1]
|
tl_x, tl_y = bubble_bbox[0], bubble_bbox[1]
|
||||||
# Adjust offset and Y-coordinate calculation based on actual layout
|
avatar_x = tl_x + offset_x
|
||||||
avatar_x = tl_x + AVATAR_OFFSET_X # Use updated offset
|
avatar_y = tl_y # Assuming Y is same as top-left
|
||||||
avatar_y = tl_y # Assume Y coordinate is same as top-left
|
# print(f"Calculated avatar coordinates: ({int(avatar_x)}, {int(avatar_y)})") # Reduce noise
|
||||||
print(f"Calculated avatar coordinates: ({int(avatar_x)}, {int(avatar_y)})")
|
return (int(avatar_x), int(avatar_y))
|
||||||
return (avatar_x, avatar_y)
|
|
||||||
|
|
||||||
def get_bubble_text(keyword_coords):
|
def get_current_ui_state(self) -> str:
|
||||||
"""
|
"""Determine the current UI state based on visible elements."""
|
||||||
Click on keyword position, simulate menu selection "Copy" or press Ctrl+C, and get text from clipboard.
|
# Check in order of specificity or likelihood
|
||||||
"""
|
if self._find_template('profile_name_page', confidence=self.state_confidence):
|
||||||
print(f"Attempting to copy @ {keyword_coords}...");
|
return 'user_details'
|
||||||
original_clipboard = get_clipboard_text() or "" # Ensure not None
|
if self._find_template('profile_page', confidence=self.state_confidence):
|
||||||
set_clipboard_text("___MCP_CLEAR___") # Use special marker to clear
|
return 'profile_card'
|
||||||
time.sleep(0.1) # Brief wait for clipboard operation
|
# Add checks for world/private chat later
|
||||||
|
if self._find_template('world_chat', confidence=self.state_confidence): # Example
|
||||||
|
return 'world_chat'
|
||||||
|
if self._find_template('private_chat', confidence=self.state_confidence): # Example
|
||||||
|
return 'private_chat'
|
||||||
|
if self._find_template('chat_room', confidence=self.state_confidence):
|
||||||
|
return 'chat_room' # General chat room if others aren't found
|
||||||
|
|
||||||
# Click on keyword position
|
return 'unknown'
|
||||||
click_at(keyword_coords[0], keyword_coords[1])
|
|
||||||
time.sleep(0.2) # Wait for possible menu or reaction
|
|
||||||
|
|
||||||
# Try to find and click "Copy" menu item
|
# ==============================================================================
|
||||||
copy_item_locations = find_template_on_screen(COPY_MENU_ITEM_IMG, confidence=0.7)
|
# Interaction Module
|
||||||
copied = False # Initialize copy state
|
# ==============================================================================
|
||||||
|
class InteractionModule:
|
||||||
|
"""Handles performing actions on the UI like clicking, typing, clipboard."""
|
||||||
|
|
||||||
|
def __init__(self, detector: DetectionModule, input_coords: Tuple[int, int] = (CHAT_INPUT_CENTER_X, CHAT_INPUT_CENTER_Y), input_template_key: Optional[str] = 'chat_input', send_button_key: str = 'send_button'):
|
||||||
|
self.detector = detector
|
||||||
|
self.default_input_coords = input_coords
|
||||||
|
self.input_template_key = input_template_key
|
||||||
|
self.send_button_key = send_button_key
|
||||||
|
print("InteractionModule initialized.")
|
||||||
|
|
||||||
|
def click_at(self, x: int, y: int, button: str = 'left', clicks: int = 1, interval: float = 0.1, duration: float = 0.1):
|
||||||
|
"""Safely click at specific coordinates."""
|
||||||
|
try:
|
||||||
|
print(f"Moving to and clicking at: ({x}, {y}), button: {button}, clicks: {clicks}")
|
||||||
|
pyautogui.moveTo(x, y, duration=duration)
|
||||||
|
pyautogui.click(button=button, clicks=clicks, interval=interval)
|
||||||
|
time.sleep(0.1)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"Error clicking at coordinates ({x}, {y}): {e}")
|
||||||
|
|
||||||
|
def press_key(self, key: str, presses: int = 1, interval: float = 0.1):
|
||||||
|
"""Press a specific key."""
|
||||||
|
try:
|
||||||
|
print(f"Pressing key: {key} ({presses} times)")
|
||||||
|
for _ in range(presses):
|
||||||
|
pyautogui.press(key)
|
||||||
|
time.sleep(interval)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"Error pressing key '{key}': {e}")
|
||||||
|
|
||||||
|
def hotkey(self, *args):
|
||||||
|
"""Press a key combination (e.g., 'ctrl', 'c')."""
|
||||||
|
try:
|
||||||
|
print(f"Pressing hotkey: {args}")
|
||||||
|
pyautogui.hotkey(*args)
|
||||||
|
time.sleep(0.1) # Short pause after hotkey
|
||||||
|
except Exception as e:
|
||||||
|
print(f"Error pressing hotkey {args}: {e}")
|
||||||
|
|
||||||
|
def get_clipboard(self) -> Optional[str]:
|
||||||
|
"""Get text from clipboard."""
|
||||||
|
try:
|
||||||
|
return pyperclip.paste()
|
||||||
|
except Exception as e:
|
||||||
|
print(f"Error reading clipboard: {e}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
def set_clipboard(self, text: str):
|
||||||
|
"""Set clipboard text."""
|
||||||
|
try:
|
||||||
|
pyperclip.copy(text)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"Error writing to clipboard: {e}")
|
||||||
|
|
||||||
|
def copy_text_at(self, coords: Tuple[int, int]) -> Optional[str]:
|
||||||
|
"""Attempt to copy text after clicking at given coordinates."""
|
||||||
|
print(f"Attempting to copy text at {coords}...")
|
||||||
|
original_clipboard = self.get_clipboard() or ""
|
||||||
|
self.set_clipboard("___MCP_CLEAR___")
|
||||||
|
time.sleep(0.1)
|
||||||
|
|
||||||
|
self.click_at(coords[0], coords[1])
|
||||||
|
time.sleep(0.2) # Wait for menu/reaction
|
||||||
|
|
||||||
|
copied = False
|
||||||
|
# Try finding "Copy" menu item first
|
||||||
|
copy_item_locations = self.detector._find_template('copy_menu_item', confidence=0.7) # Use detector
|
||||||
if copy_item_locations:
|
if copy_item_locations:
|
||||||
copy_coords = copy_item_locations[0]
|
copy_coords = copy_item_locations[0]
|
||||||
click_at(copy_coords[0], copy_coords[1])
|
self.click_at(copy_coords[0], copy_coords[1])
|
||||||
print("Clicked 'Copy' menu item.")
|
print("Clicked 'Copy' menu item.")
|
||||||
time.sleep(0.2) # Wait for copy operation to complete
|
time.sleep(0.2)
|
||||||
copied = True # Mark copy operation as attempted (via click)
|
copied = True
|
||||||
else:
|
else:
|
||||||
print("'Copy' menu item not found. Attempting to simulate Ctrl+C.")
|
print("'Copy' menu item not found. Attempting Ctrl+C.")
|
||||||
# --- Corrected try block ---
|
|
||||||
try:
|
try:
|
||||||
pyautogui.hotkey('ctrl', 'c')
|
self.hotkey('ctrl', 'c')
|
||||||
time.sleep(0.2) # Wait for copy operation to complete
|
time.sleep(0.2)
|
||||||
print("Simulated Ctrl+C.")
|
print("Simulated Ctrl+C.")
|
||||||
copied = True # Mark copy operation as attempted (via hotkey)
|
copied = True
|
||||||
except Exception as e_ctrlc:
|
except Exception as e_ctrlc:
|
||||||
print(f"Failed to simulate Ctrl+C: {e_ctrlc}")
|
print(f"Failed to simulate Ctrl+C: {e_ctrlc}")
|
||||||
copied = False # Ensure copied is False on failure
|
copied = False
|
||||||
# --- End correction ---
|
|
||||||
|
|
||||||
# Check clipboard content
|
copied_text = self.get_clipboard()
|
||||||
copied_text = get_clipboard_text()
|
self.set_clipboard(original_clipboard) # Restore clipboard
|
||||||
|
|
||||||
# Restore original clipboard
|
|
||||||
pyperclip.copy(original_clipboard)
|
|
||||||
|
|
||||||
# Determine if copy was successful
|
|
||||||
if copied and copied_text and copied_text != "___MCP_CLEAR___":
|
if copied and copied_text and copied_text != "___MCP_CLEAR___":
|
||||||
print(f"Successfully copied text, length: {len(copied_text)}")
|
print(f"Successfully copied text, length: {len(copied_text)}")
|
||||||
return copied_text.strip() # Return text with leading/trailing whitespace removed
|
return copied_text.strip()
|
||||||
else:
|
else:
|
||||||
print("Error: Copy operation unsuccessful or clipboard content invalid.")
|
print("Error: Copy operation unsuccessful or clipboard content invalid.")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def get_sender_name(avatar_coords):
|
def retrieve_sender_name_interaction(self, avatar_coords: Tuple[int, int]) -> Optional[str]:
|
||||||
"""
|
"""
|
||||||
Click avatar, open profile card, click option, open user details, click copy name.
|
Perform the sequence of actions to copy sender name, *without* cleanup.
|
||||||
Uses state-based ESC cleanup logic.
|
Returns the name or None if failed.
|
||||||
"""
|
"""
|
||||||
print(f"Attempting to get username from avatar {avatar_coords}...")
|
print(f"Attempting interaction to get username from avatar {avatar_coords}...")
|
||||||
original_clipboard = get_clipboard_text() or ""
|
original_clipboard = self.get_clipboard() or ""
|
||||||
set_clipboard_text("___MCP_CLEAR___")
|
self.set_clipboard("___MCP_CLEAR___")
|
||||||
time.sleep(0.1)
|
time.sleep(0.1)
|
||||||
sender_name = None # Initialize
|
sender_name = None
|
||||||
success = False # Mark whether name retrieval was successful
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# 1. Click avatar
|
# 1. Click avatar
|
||||||
click_at(avatar_coords[0], avatar_coords[1])
|
self.click_at(avatar_coords[0], avatar_coords[1])
|
||||||
time.sleep(.3) # Wait for profile card to appear
|
time.sleep(0.3) # Wait for profile card
|
||||||
|
|
||||||
# 2. Find and click option on profile card (triggers user details)
|
# 2. Find and click profile option
|
||||||
profile_option_locations = find_template_on_screen(PROFILE_OPTION_IMG, confidence=0.7)
|
profile_option_locations = self.detector._find_template('profile_option', confidence=0.7)
|
||||||
if not profile_option_locations:
|
if not profile_option_locations:
|
||||||
print("Error: User details option not found on profile card.")
|
print("Error: User details option not found on profile card.")
|
||||||
# No need to raise exception here, let finally handle cleanup
|
return None # Fail early if critical step missing
|
||||||
else:
|
self.click_at(profile_option_locations[0][0], profile_option_locations[0][1])
|
||||||
click_at(profile_option_locations[0][0], profile_option_locations[0][1])
|
|
||||||
print("Clicked user details option.")
|
print("Clicked user details option.")
|
||||||
time.sleep(.3) # Wait for user details window to appear
|
time.sleep(0.3) # Wait for user details window
|
||||||
|
|
||||||
# 3. Find and click "Copy Name" button in user details
|
# 3. Find and click "Copy Name" button
|
||||||
copy_name_locations = find_template_on_screen(COPY_NAME_BUTTON_IMG, confidence=0.7)
|
copy_name_locations = self.detector._find_template('copy_name_button', confidence=0.7)
|
||||||
if not copy_name_locations:
|
if not copy_name_locations:
|
||||||
print("Error: 'Copy Name' button not found in user details.")
|
print("Error: 'Copy Name' button not found in user details.")
|
||||||
else:
|
return None # Fail early
|
||||||
click_at(copy_name_locations[0][0], copy_name_locations[0][1])
|
self.click_at(copy_name_locations[0][0], copy_name_locations[0][1])
|
||||||
print("Clicked 'Copy Name' button.")
|
print("Clicked 'Copy Name' button.")
|
||||||
time.sleep(0.1) # Wait for copy to complete
|
time.sleep(0.1)
|
||||||
copied_name = get_clipboard_text()
|
|
||||||
|
# 4. Get name from clipboard
|
||||||
|
copied_name = self.get_clipboard()
|
||||||
if copied_name and copied_name != "___MCP_CLEAR___":
|
if copied_name and copied_name != "___MCP_CLEAR___":
|
||||||
print(f"Successfully copied username: {copied_name}")
|
print(f"Successfully copied username: {copied_name}")
|
||||||
sender_name = copied_name.strip() # Store successfully copied name
|
sender_name = copied_name.strip()
|
||||||
success = True # Mark success
|
|
||||||
else:
|
else:
|
||||||
print("Error: Clipboard content unchanged or empty, failed to copy username.")
|
print("Error: Clipboard content invalid after clicking copy name.")
|
||||||
|
sender_name = None
|
||||||
|
|
||||||
# Regardless of success above, return sender_name (might be None)
|
|
||||||
return sender_name
|
return sender_name
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"Error during username retrieval process: {e}")
|
print(f"Error during username retrieval interaction: {e}")
|
||||||
import traceback
|
import traceback
|
||||||
traceback.print_exc()
|
traceback.print_exc()
|
||||||
return None # Return None to indicate failure
|
return None
|
||||||
finally:
|
finally:
|
||||||
# --- State-based cleanup logic ---
|
# Restore clipboard regardless of success/failure
|
||||||
print("Performing cleanup: Attempting to press ESC to return to chat interface based on screen state...")
|
self.set_clipboard(original_clipboard)
|
||||||
max_esc_attempts = 4 # Increase attempt count just in case
|
# NO cleanup logic here - should be handled by coordinator
|
||||||
returned_to_chat = False
|
|
||||||
for attempt in range(max_esc_attempts):
|
|
||||||
print(f"Cleanup attempt #{attempt + 1}/{max_esc_attempts}")
|
|
||||||
time.sleep(0.2) # Short wait before each attempt
|
|
||||||
|
|
||||||
# First check if already returned to chat room
|
def send_chat_message(self, reply_text: str) -> bool:
|
||||||
# Using lower confidence for state checks may be more stable
|
"""Paste text into chat input and send it."""
|
||||||
if find_template_on_screen(CHAT_ROOM_IMG, confidence=STATE_CONFIDENCE_THRESHOLD):
|
|
||||||
print("Chat room interface detected, cleanup complete.")
|
|
||||||
returned_to_chat = True
|
|
||||||
break # Already returned, exit loop
|
|
||||||
|
|
||||||
# Check if in user details page
|
|
||||||
elif find_template_on_screen(PROFILE_NAME_PAGE_IMG, confidence=STATE_CONFIDENCE_THRESHOLD):
|
|
||||||
print("User details page detected, pressing ESC...")
|
|
||||||
pyautogui.press('esc')
|
|
||||||
time.sleep(0.2) # Wait for UI response
|
|
||||||
continue # Continue to next loop iteration
|
|
||||||
|
|
||||||
# Check if in profile card page
|
|
||||||
elif find_template_on_screen(PROFILE_PAGE_IMG, confidence=STATE_CONFIDENCE_THRESHOLD):
|
|
||||||
print("Profile card page detected, pressing ESC...")
|
|
||||||
pyautogui.press('esc')
|
|
||||||
time.sleep(0.2) # Wait for UI response
|
|
||||||
continue # Continue to next loop iteration
|
|
||||||
|
|
||||||
else:
|
|
||||||
# Cannot identify current state
|
|
||||||
print("No known page state detected.")
|
|
||||||
if attempt < max_esc_attempts - 1:
|
|
||||||
print("Trying one ESC press as fallback...")
|
|
||||||
pyautogui.press('esc')
|
|
||||||
time.sleep(0.2) # Wait for response
|
|
||||||
else:
|
|
||||||
print("Maximum attempts reached, stopping cleanup.")
|
|
||||||
break # Exit loop
|
|
||||||
|
|
||||||
if not returned_to_chat:
|
|
||||||
print("Warning: Could not confirm return to chat room interface via state detection.")
|
|
||||||
# --- End of new cleanup logic ---
|
|
||||||
|
|
||||||
# Ensure clipboard is restored
|
|
||||||
pyperclip.copy(original_clipboard)
|
|
||||||
|
|
||||||
|
|
||||||
def paste_and_send_reply(reply_text):
|
|
||||||
"""
|
|
||||||
Click chat input box, paste response, click send button or press Enter.
|
|
||||||
"""
|
|
||||||
print("Preparing to send response...")
|
print("Preparing to send response...")
|
||||||
# --- Corrected if statement ---
|
|
||||||
if not reply_text:
|
if not reply_text:
|
||||||
print("Error: Response content is empty, cannot send.")
|
print("Error: Response content is empty, cannot send.")
|
||||||
return False
|
return False
|
||||||
# --- End correction ---
|
|
||||||
|
|
||||||
input_coords = None
|
# Find input box coordinates
|
||||||
if os.path.exists(CHAT_INPUT_IMG):
|
input_coords = self.default_input_coords # Fallback
|
||||||
input_locations = find_template_on_screen(CHAT_INPUT_IMG, confidence=0.7)
|
if self.input_template_key and self.detector.templates.get(self.input_template_key):
|
||||||
|
input_locations = self.detector._find_template(self.input_template_key, confidence=0.7)
|
||||||
if input_locations:
|
if input_locations:
|
||||||
input_coords = input_locations[0]
|
input_coords = input_locations[0]
|
||||||
print(f"Found input box position via image: {input_coords}")
|
print(f"Found input box position via image: {input_coords}")
|
||||||
else:
|
else:
|
||||||
print("Warning: Input box not found via image, using default coordinates.")
|
print(f"Warning: Input box template '{self.input_template_key}' not found, using default coordinates.")
|
||||||
input_coords = (CHAT_INPUT_CENTER_X, CHAT_INPUT_CENTER_Y)
|
|
||||||
else:
|
else:
|
||||||
print("Warning: Input box template image doesn't exist, using default coordinates.")
|
print("Warning: Input box template key not set or image missing, using default coordinates.")
|
||||||
input_coords = (CHAT_INPUT_CENTER_X, CHAT_INPUT_CENTER_Y)
|
|
||||||
|
|
||||||
click_at(input_coords[0], input_coords[1])
|
# Click input, paste, send
|
||||||
|
self.click_at(input_coords[0], input_coords[1])
|
||||||
time.sleep(0.3)
|
time.sleep(0.3)
|
||||||
|
|
||||||
print("Pasting response...")
|
print("Pasting response...")
|
||||||
set_clipboard_text(reply_text)
|
self.set_clipboard(reply_text)
|
||||||
time.sleep(0.1)
|
time.sleep(0.1)
|
||||||
try:
|
try:
|
||||||
pyautogui.hotkey('ctrl', 'v')
|
self.hotkey('ctrl', 'v')
|
||||||
time.sleep(0.5)
|
time.sleep(0.5)
|
||||||
print("Pasted.")
|
print("Pasted.")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"Error pasting response: {e}")
|
print(f"Error pasting response: {e}")
|
||||||
return False
|
return False
|
||||||
|
|
||||||
send_button_locations = find_template_on_screen(SEND_BUTTON_IMG, confidence=0.7)
|
# Try clicking send button first
|
||||||
|
send_button_locations = self.detector._find_template(self.send_button_key, confidence=0.7)
|
||||||
if send_button_locations:
|
if send_button_locations:
|
||||||
send_coords = send_button_locations[0]
|
send_coords = send_button_locations[0]
|
||||||
click_at(send_coords[0], send_coords[1])
|
self.click_at(send_coords[0], send_coords[1])
|
||||||
print("Clicked send button.")
|
print("Clicked send button.")
|
||||||
time.sleep(0.1)
|
time.sleep(0.1)
|
||||||
return True
|
return True
|
||||||
else:
|
else:
|
||||||
|
# Fallback to pressing Enter
|
||||||
print("Send button not found. Attempting to press Enter.")
|
print("Send button not found. Attempting to press Enter.")
|
||||||
try:
|
try:
|
||||||
pyautogui.press('enter')
|
self.press_key('enter')
|
||||||
print("Pressed Enter.")
|
print("Pressed Enter.")
|
||||||
time.sleep(0.5)
|
time.sleep(0.5)
|
||||||
return True
|
return True
|
||||||
@ -455,51 +418,187 @@ def paste_and_send_reply(reply_text):
|
|||||||
print(f"Error pressing Enter: {e_enter}")
|
print(f"Error pressing Enter: {e_enter}")
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
# ==============================================================================
|
||||||
|
# Coordinator Logic (Placeholder - To be implemented in main.py)
|
||||||
|
# ==============================================================================
|
||||||
|
|
||||||
# --- Main Monitoring and Triggering Logic ---
|
# --- State-based Cleanup Function (To be called by Coordinator) ---
|
||||||
recent_texts = collections.deque(maxlen=RECENT_TEXT_HISTORY_MAXLEN)
|
def perform_state_cleanup(detector: DetectionModule, interactor: InteractionModule, max_attempts: int = 4) -> bool:
|
||||||
|
"""
|
||||||
|
Attempt to return to the main chat room interface by pressing ESC based on detected state.
|
||||||
|
Returns True if confirmed back in chat room, False otherwise.
|
||||||
|
"""
|
||||||
|
print("Performing cleanup: Attempting to press ESC to return to chat interface...")
|
||||||
|
returned_to_chat = False
|
||||||
|
for attempt in range(max_attempts):
|
||||||
|
print(f"Cleanup attempt #{attempt + 1}/{max_attempts}")
|
||||||
|
time.sleep(0.2)
|
||||||
|
|
||||||
|
current_state = detector.get_current_ui_state()
|
||||||
|
print(f"Detected state: {current_state}")
|
||||||
|
|
||||||
|
if current_state == 'chat_room' or current_state == 'world_chat' or current_state == 'private_chat': # Adjust as needed
|
||||||
|
print("Chat room interface detected, cleanup complete.")
|
||||||
|
returned_to_chat = True
|
||||||
|
break
|
||||||
|
elif current_state == 'user_details' or current_state == 'profile_card':
|
||||||
|
print(f"{current_state.replace('_', ' ').title()} detected, pressing ESC...")
|
||||||
|
interactor.press_key('esc')
|
||||||
|
time.sleep(0.3) # Wait longer for UI response after ESC
|
||||||
|
continue
|
||||||
|
else: # Unknown state
|
||||||
|
print("Unknown page state detected.")
|
||||||
|
if attempt < max_attempts - 1:
|
||||||
|
print("Trying one ESC press as fallback...")
|
||||||
|
interactor.press_key('esc')
|
||||||
|
time.sleep(0.3)
|
||||||
|
else:
|
||||||
|
print("Maximum attempts reached, stopping cleanup.")
|
||||||
|
break
|
||||||
|
|
||||||
|
if not returned_to_chat:
|
||||||
|
print("Warning: Could not confirm return to chat room interface via state detection.")
|
||||||
|
return returned_to_chat
|
||||||
|
|
||||||
|
|
||||||
|
# --- UI Monitoring Loop Function (To be run in a separate thread) ---
|
||||||
|
def run_ui_monitoring_loop(trigger_queue: queue.Queue, command_queue: queue.Queue):
|
||||||
|
"""
|
||||||
|
Continuously monitors the UI, detects triggers, performs interactions,
|
||||||
|
puts trigger data into trigger_queue, and processes commands from command_queue.
|
||||||
|
"""
|
||||||
|
print("\n--- Starting UI Monitoring Loop (Thread) ---")
|
||||||
|
|
||||||
|
# --- Initialization (Instantiate modules within the thread) ---
|
||||||
|
# Load templates directly using constants defined in this file for now
|
||||||
|
# Consider passing config or a template loader object in the future
|
||||||
|
templates = {
|
||||||
|
'corner_tl': CORNER_TL_IMG, 'corner_br': CORNER_BR_IMG,
|
||||||
|
'bot_corner_tl': BOT_CORNER_TL_IMG, 'bot_corner_br': BOT_CORNER_BR_IMG,
|
||||||
|
'keyword_wolf_lower': KEYWORD_wolf_LOWER_IMG, 'keyword_wolf_upper': KEYWORD_Wolf_UPPER_IMG,
|
||||||
|
'copy_menu_item': COPY_MENU_ITEM_IMG, 'profile_option': PROFILE_OPTION_IMG,
|
||||||
|
'copy_name_button': COPY_NAME_BUTTON_IMG, 'send_button': SEND_BUTTON_IMG,
|
||||||
|
'chat_input': CHAT_INPUT_IMG, 'profile_name_page': PROFILE_NAME_PAGE_IMG,
|
||||||
|
'profile_page': PROFILE_PAGE_IMG, 'chat_room': CHAT_ROOM_IMG,
|
||||||
|
'world_chat': WORLD_CHAT_IMG, 'private_chat': PRIVATE_CHAT_IMG # Add other templates as needed
|
||||||
|
}
|
||||||
|
# Use default confidence/region settings from constants
|
||||||
|
detector = DetectionModule(templates, confidence=CONFIDENCE_THRESHOLD, state_confidence=STATE_CONFIDENCE_THRESHOLD, region=SCREENSHOT_REGION)
|
||||||
|
# Use default input coords/keys from constants
|
||||||
|
interactor = InteractionModule(detector, input_coords=(CHAT_INPUT_CENTER_X, CHAT_INPUT_CENTER_Y), input_template_key='chat_input', send_button_key='send_button')
|
||||||
|
|
||||||
|
# --- State Management (Local to this monitoring thread) ---
|
||||||
last_processed_bubble_bbox = None
|
last_processed_bubble_bbox = None
|
||||||
|
recent_texts = collections.deque(maxlen=RECENT_TEXT_HISTORY_MAXLEN) # Context-specific history needed
|
||||||
def monitor_chat_for_trigger(trigger_queue: queue.Queue): # Using standard queue
|
|
||||||
"""
|
|
||||||
Continuously monitor chat area, look for bubbles containing keywords and put trigger info in Queue.
|
|
||||||
"""
|
|
||||||
global last_processed_bubble_bbox
|
|
||||||
print(f"\n--- Starting chat room monitoring (UI Thread) ---")
|
|
||||||
# No longer need to get loop
|
|
||||||
|
|
||||||
while True:
|
while True:
|
||||||
|
# --- Process Commands First (Non-blocking) ---
|
||||||
try:
|
try:
|
||||||
all_bubbles_with_type = find_dialogue_bubbles()
|
command_data = command_queue.get_nowait() # Check for commands without blocking
|
||||||
if not all_bubbles_with_type: time.sleep(2); continue
|
action = command_data.get('action')
|
||||||
other_bubbles_bboxes = [bbox for bbox, is_bot in all_bubbles_with_type if not is_bot]
|
if action == 'send_reply':
|
||||||
if not other_bubbles_bboxes: time.sleep(2); continue
|
text_to_send = command_data.get('text')
|
||||||
target_bubble = max(other_bubbles_bboxes, key=lambda b: b[3])
|
if text_to_send:
|
||||||
if are_bboxes_similar(target_bubble, last_processed_bubble_bbox): time.sleep(2); continue
|
print(f"UI Thread: Received command to send reply: '{text_to_send[:50]}...'")
|
||||||
|
interactor.send_chat_message(text_to_send)
|
||||||
|
else:
|
||||||
|
print("UI Thread: Received send_reply command with no text.")
|
||||||
|
else:
|
||||||
|
print(f"UI Thread: Received unknown command: {action}")
|
||||||
|
except queue.Empty:
|
||||||
|
pass # No command waiting, continue with monitoring
|
||||||
|
except Exception as cmd_err:
|
||||||
|
print(f"UI Thread: Error processing command queue: {cmd_err}")
|
||||||
|
|
||||||
|
# --- Then Perform UI Monitoring ---
|
||||||
|
try:
|
||||||
|
# 1. Detect Bubbles
|
||||||
|
all_bubbles = detector.find_dialogue_bubbles()
|
||||||
|
if not all_bubbles: time.sleep(2); continue
|
||||||
|
|
||||||
|
# Filter out bot bubbles, find newest non-bot bubble (example logic)
|
||||||
|
other_bubbles = [bbox for bbox, is_bot in all_bubbles if not is_bot]
|
||||||
|
if not other_bubbles: time.sleep(2); continue
|
||||||
|
# Simple logic: assume lowest bubble is newest (might need improvement)
|
||||||
|
target_bubble = max(other_bubbles, key=lambda b: b[3]) # b[3] is y_max
|
||||||
|
|
||||||
|
# 2. Check for Duplicates (Position & Content)
|
||||||
|
if are_bboxes_similar(target_bubble, last_processed_bubble_bbox):
|
||||||
|
time.sleep(2); continue
|
||||||
|
|
||||||
|
# 3. Detect Keyword in Bubble
|
||||||
|
bubble_region = (target_bubble[0], target_bubble[1], target_bubble[2]-target_bubble[0], target_bubble[3]-target_bubble[1])
|
||||||
|
keyword_coords = detector.find_keyword_in_region(bubble_region)
|
||||||
|
|
||||||
keyword_coords = find_keyword_in_bubble(target_bubble)
|
|
||||||
if keyword_coords:
|
if keyword_coords:
|
||||||
print(f"\n!!! Keyword detected in bubble {target_bubble} !!!")
|
print(f"\n!!! Keyword detected in bubble {target_bubble} !!!")
|
||||||
bubble_text = get_bubble_text(keyword_coords) # Using corrected version
|
|
||||||
if not bubble_text: print("Error: Could not get dialogue content."); last_processed_bubble_bbox = target_bubble; continue
|
|
||||||
if bubble_text in recent_texts: print(f"Content '{bubble_text[:30]}...' in recent history, skipping."); last_processed_bubble_bbox = target_bubble; continue
|
|
||||||
|
|
||||||
print(">>> New trigger event <<<"); last_processed_bubble_bbox = target_bubble; recent_texts.append(bubble_text)
|
# 4. Interact: Get Bubble Text
|
||||||
avatar_coords = find_avatar_for_bubble(target_bubble)
|
bubble_text = interactor.copy_text_at(keyword_coords)
|
||||||
sender_name = get_sender_name(avatar_coords) # Using version with state cleanup
|
if not bubble_text:
|
||||||
if not sender_name: print("Error: Could not get sender name, aborting processing."); continue
|
print("Error: Could not get dialogue content.")
|
||||||
|
last_processed_bubble_bbox = target_bubble # Mark as processed even if failed
|
||||||
|
perform_state_cleanup(detector, interactor) # Attempt cleanup after failed copy
|
||||||
|
continue
|
||||||
|
|
||||||
print("\n>>> Putting trigger info in Queue <<<"); print(f" Sender: {sender_name}"); print(f" Content: {bubble_text[:100]}...")
|
# Check recent text history (needs context awareness)
|
||||||
|
if bubble_text in recent_texts:
|
||||||
|
print(f"Content '{bubble_text[:30]}...' in recent history, skipping.")
|
||||||
|
last_processed_bubble_bbox = target_bubble
|
||||||
|
continue
|
||||||
|
|
||||||
|
print(">>> New trigger event <<<")
|
||||||
|
last_processed_bubble_bbox = target_bubble
|
||||||
|
recent_texts.append(bubble_text)
|
||||||
|
|
||||||
|
# 5. Interact: Get Sender Name
|
||||||
|
avatar_coords = detector.calculate_avatar_coords(target_bubble)
|
||||||
|
sender_name = interactor.retrieve_sender_name_interaction(avatar_coords)
|
||||||
|
|
||||||
|
# 6. Perform Cleanup (Crucial after potentially leaving chat screen)
|
||||||
|
cleanup_successful = perform_state_cleanup(detector, interactor)
|
||||||
|
if not cleanup_successful:
|
||||||
|
print("Error: Failed to return to chat screen after getting name. Aborting trigger.")
|
||||||
|
continue # Skip putting in queue if cleanup failed
|
||||||
|
|
||||||
|
if not sender_name:
|
||||||
|
print("Error: Could not get sender name, aborting processing.")
|
||||||
|
continue # Already cleaned up, just skip
|
||||||
|
|
||||||
|
# 7. Send Trigger Info to Main Thread/Async Loop
|
||||||
|
print("\n>>> Putting trigger info in Queue <<<")
|
||||||
|
print(f" Sender: {sender_name}")
|
||||||
|
print(f" Content: {bubble_text[:100]}...")
|
||||||
try:
|
try:
|
||||||
# --- Using queue.put (synchronous) ---
|
|
||||||
data_to_send = {'sender': sender_name, 'text': bubble_text}
|
data_to_send = {'sender': sender_name, 'text': bubble_text}
|
||||||
trigger_queue.put(data_to_send) # Directly put into standard Queue
|
trigger_queue.put(data_to_send) # Put in the queue for main loop
|
||||||
print("Trigger info placed in Queue.")
|
print("Trigger info placed in Queue.")
|
||||||
except Exception as q_err: print(f"Error putting data in Queue: {q_err}")
|
except Exception as q_err:
|
||||||
print("--- Single trigger processing complete ---"); time.sleep(1)
|
print(f"Error putting data in Queue: {q_err}")
|
||||||
time.sleep(1.5)
|
|
||||||
except KeyboardInterrupt: print("\nMonitoring interrupted."); break
|
|
||||||
except Exception as e: print(f"Unknown error in monitoring loop: {e}"); import traceback; traceback.print_exc(); print("Waiting 5 seconds before retry..."); time.sleep(5)
|
|
||||||
|
|
||||||
# if __name__ == '__main__': # Keep commented, typically called from main.py
|
print("--- Single trigger processing complete ---")
|
||||||
|
time.sleep(1) # Pause after successful trigger
|
||||||
|
|
||||||
|
time.sleep(1.5) # Polling interval
|
||||||
|
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
print("\nMonitoring interrupted.")
|
||||||
|
break
|
||||||
|
except Exception as e:
|
||||||
|
print(f"Unknown error in monitoring loop: {e}")
|
||||||
|
import traceback
|
||||||
|
traceback.print_exc()
|
||||||
|
# Attempt cleanup in case of unexpected error during interaction
|
||||||
|
print("Attempting cleanup after unexpected error...")
|
||||||
|
perform_state_cleanup(detector, interactor)
|
||||||
|
print("Waiting 5 seconds before retry...")
|
||||||
|
time.sleep(5)
|
||||||
|
|
||||||
|
# Note: The old monitor_chat_for_trigger function is replaced by the example_coordinator_loop.
|
||||||
|
# The actual UI monitoring thread started in main.py should call a function like this example loop.
|
||||||
|
# The main async loop in main.py will handle getting items from the queue and interacting with the LLM.
|
||||||
|
|
||||||
|
# if __name__ == '__main__':
|
||||||
|
# # This module is not meant to be run directly after refactoring.
|
||||||
|
# # Initialization and coordination happen in main.py.
|
||||||
# pass
|
# pass
|
||||||
|
|||||||