feat: Add persistent message deduplication

- Fix duplicate message processing during unattended operation
- Replace MessageDeduplication with persistent RobustMessageDeduplication
- Add automatic cleanup of expired records (every 5 minutes)
- Store deduplication state in JSON file to survive system restarts
- Improve similarity detection (95% threshold) for near-duplicate messages

.gitignore

@@ -10,4 +10,5 @@ chat_logs/
 backup/
 chroma_data/
 wolf_control.py
-remote_config.json
+remote_config.json
+wolf_chat_dedup.json

main.py

@@ -16,8 +16,8 @@ from mcp import ClientSession, StdioServerParameters, types
 # --- Keyboard Imports ---
 import threading
 import time
-# Import MessageDeduplication from ui_interaction
+# Import RobustMessageDeduplication and StateResetDetector from ui_interaction
-from ui_interaction import MessageDeduplication
+from ui_interaction import RobustMessageDeduplication, StateResetDetector 
 try:
     import keyboard # Needs pip install keyboard
 except ImportError:
@@ -483,33 +483,45 @@ async def run_main_with_exit_stack():
 
         # 5. Start UI Monitoring in a separate thread
         print("\n--- Starting UI monitoring thread ---")
-        # 5c. Create MessageDeduplication instance
+        
-        deduplicator = MessageDeduplication(expiry_seconds=3600) # Default 1 hour
+        # 5c. Initialize Robust Deduplication System
+        def initialize_robust_deduplication():
+            """初始化強化版去重系統"""
+            deduplicator_instance = RobustMessageDeduplication(
+                storage_file="wolf_chat_dedup.json",
+                expiry_seconds=3600  # 1小時過期
+            )
+            state_monitor_instance = StateResetDetector("wolf_chat_state_resets.log")
+            return deduplicator_instance, state_monitor_instance
 
-        # Use the new monitoring loop function, passing both queues and the deduplicator
+        deduplicator, state_monitor = initialize_robust_deduplication()
+
+        # Use the new monitoring loop function, passing trigger_queue, command_queue, deduplicator, and state_monitor
         monitor_task = loop.create_task(
-            asyncio.to_thread(ui_interaction.run_ui_monitoring_loop, trigger_queue, command_queue, deduplicator), # Pass command_queue and deduplicator
+            asyncio.to_thread(ui_interaction.run_ui_monitoring_loop_enhanced, trigger_queue, command_queue, deduplicator, state_monitor),
-            name="ui_monitor"
+            name="ui_monitor_enhanced"
         )
         ui_monitor_task = monitor_task # Store task reference for shutdown
         # Note: UI task cancellation is handled in shutdown()
 
         # 5b. Game Window Monitoring is now handled by Setup.py
 
-        # 5d. Start Periodic Cleanup Timer for Deduplicator
+        # 5d. Start Periodic Cleanup and Stats Logging Timer for Deduplicator
-        def periodic_cleanup():
+        def periodic_robust_cleanup_and_stats():
             if not shutdown_requested: # Only run if not shutting down
-                print("Main Thread: Running periodic deduplicator cleanup...")
+                print("Main Thread: Running periodic robust deduplicator cleanup and stats logging...")
-                deduplicator.purge_expired()
+                deduplicator._cleanup_expired() # Call internal cleanup
+                stats = deduplicator.get_stats()
+                print(f"Main Thread - Dedup Stats: {stats['active_records']} active records (total: {stats['total_records']})")
                 # Reschedule the timer
-                cleanup_timer = threading.Timer(600, periodic_cleanup) # 10 minutes
+                cleanup_timer = threading.Timer(600, periodic_robust_cleanup_and_stats) # 10 minutes
                 cleanup_timer.daemon = True
                 cleanup_timer.start()
             else:
-                print("Main Thread: Shutdown requested, not rescheduling deduplicator cleanup.")
+                print("Main Thread: Shutdown requested, not rescheduling robust deduplicator cleanup.")
 
-        print("\n--- Starting periodic deduplicator cleanup timer (10 min interval) ---")
+        print("\n--- Starting periodic robust deduplicator cleanup and stats timer (10 min interval) ---")
-        initial_cleanup_timer = threading.Timer(600, periodic_cleanup)
+        initial_cleanup_timer = threading.Timer(600, periodic_robust_cleanup_and_stats)
         initial_cleanup_timer.daemon = True
         initial_cleanup_timer.start()
         # Note: This timer will run in a separate thread.

ui_interaction.py

@@ -21,78 +21,193 @@ import math # Added for distance calculation in dual method
 import time # Ensure time is imported for MessageDeduplication
 from simple_bubble_dedup import SimpleBubbleDeduplication
 import difflib # Added for text similarity
+import os # Already imported, but good to note for RobustMessageDeduplication
+import json # Already imported, but good to note for RobustMessageDeduplication
 
-class MessageDeduplication:
+# 替換現有的 MessageDeduplication 類
-    def __init__(self, expiry_seconds=3600):  # 1 hour expiry time
+class RobustMessageDeduplication:
-        self.processed_messages = {}  # {message_key: timestamp}
+    def __init__(self, storage_file="persistent_dedup.json", expiry_seconds=3600):
+        self.storage_file = storage_file
         self.expiry_seconds = expiry_seconds
-
+        self.processed_messages = {}
-    def is_duplicate(self, sender, content):
+        self.last_save_time = 0
-        """Check if the message is a duplicate within the expiry period using text similarity."""
+        self.save_interval = 10  # 每10秒保存一次
-        if not sender or not content:
-            return False  # Missing necessary info, treat as new message
-
-        current_time = time.time()
         
-        # 遍歷所有已處理的消息
+        # 啟動時加載持久化數據
-        for key, timestamp in list(self.processed_messages.items()):
+        self._load_from_storage()
-            # 檢查是否過期
+        
-            if current_time - timestamp >= self.expiry_seconds:
+        # 清理過期記錄
-                # 從 processed_messages 中移除過期的項目，避免集合在迭代時改變大小
+        self._cleanup_expired()
-                # 但由於我們使用了 list(self.processed_messages.items())，所以這裡可以安全地 continue
+        
-                # 或者，如果希望立即刪除，則需要不同的迭代策略或在 purge_expired 中處理
+        print(f"RobustDeduplication initialized with {len(self.processed_messages)} existing records")
-                continue # 繼續檢查下一個，過期項目由 purge_expired 處理
+    
-                
+    def _load_from_storage(self):
-            # 解析之前儲存的發送者和內容
+        """從持久化文件加載去重記錄"""
-            stored_sender, stored_content = key.split(":", 1)
+        try:
+            if os.path.exists(self.storage_file):
+                with open(self.storage_file, 'r', encoding='utf-8') as f:
+                    data = json.load(f)
+                    self.processed_messages = data.get('messages', {})
+                    print(f"Loaded {len(self.processed_messages)} dedup records from storage")
+        except Exception as e:
+            print(f"Warning: Could not load dedup storage: {e}")
+            self.processed_messages = {}
+    
+    def _save_to_storage(self, force=False):
+        """保存去重記錄到持久化文件"""
+        current_time = time.time()
+        if not force and (current_time - self.last_save_time) < self.save_interval:
+            return
+        
+        try:
+            # 只保存未過期的記錄
+            valid_records = {}
+            for key, timestamp in self.processed_messages.items():
+                if current_time - timestamp < self.expiry_seconds:
+                    valid_records[key] = timestamp
             
-            # 檢查發送者是否相同
+            data = {
-            if sender.lower() == stored_sender.lower():
+                'messages': valid_records,
-                # Calculate text similarity
+                'last_updated': current_time
-                similarity = difflib.SequenceMatcher(None, content, stored_content).ratio()
+            }
-                if similarity >= 0.95:  # Use 0.95 as threshold
+            
-                    print(f"Deduplicator: Detected similar message (similarity: {similarity:.2f}): {sender} - {content[:20]}...")
+            with open(self.storage_file, 'w', encoding='utf-8') as f:
-                    return True
+                json.dump(data, f, indent=2)
-        
+            
-        # 不是重複消息，儲存它
+            self.processed_messages = valid_records
-        # 注意：這裡儲存的 content 是原始 content，不是 clean_content
+            self.last_save_time = current_time
-        message_key = f"{sender.lower()}:{content}"
+            
-        self.processed_messages[message_key] = current_time
+        except Exception as e:
-        return False
+            print(f"Error saving dedup storage: {e}")
-
+    
-    # create_key 方法已不再需要，可以移除
+    def _cleanup_expired(self):
-    # def create_key(self, sender, content):
+        """清理過期記錄"""
-    #     """Create a standardized composite key."""
-    #     # Thoroughly standardize text - remove all whitespace and punctuation, lowercase
-    #     clean_content = ''.join(c.lower() for c in content if c.isalnum())
-    #     clean_sender = ''.join(c.lower() for c in sender if c.isalnum())
-
-    #     # Truncate content to first 100 chars to prevent overly long keys
-    #     if len(clean_content) > 100:
-    #         clean_content = clean_content[:100]
-
-    #     return f"{clean_sender}:{clean_content}"
-
-    def purge_expired(self):
-        """Remove expired message records."""
         current_time = time.time()
-        expired_keys = [k for k, t in self.processed_messages.items()
+        before_count = len(self.processed_messages)
-                         if current_time - t >= self.expiry_seconds]
+        
-
+        self.processed_messages = {
-        for key in expired_keys:
+            key: timestamp for key, timestamp in self.processed_messages.items()
-            del self.processed_messages[key]
+            if current_time - timestamp < self.expiry_seconds
-
+        }
-        if expired_keys: # Log only if something was purged
+        
-            print(f"Deduplicator: Purged {len(expired_keys)} expired message records.")
+        after_count = len(self.processed_messages)
-        return len(expired_keys)
+        if before_count > after_count:
-
+            print(f"Cleaned up {before_count - after_count} expired dedup records")
+    
+    def _create_message_key(self, sender, content):
+        """創建標準化的消息鍵"""
+        # 標準化處理
+        clean_sender = sender.lower().strip() if sender else ""
+        clean_content = ' '.join(content.strip().split()) if content else ""
+        return f"{clean_sender}:{clean_content}"
+    
+    def is_duplicate(self, sender, content):
+        """強化的重複檢查"""
+        if not sender or not content:
+            print("Deduplication: Missing sender or content, treating as new")
+            return False
+        
+        current_time = time.time()
+        
+        # 定期清理（每5分鐘）
+        if hasattr(self, '_last_cleanup'):
+            if current_time - self._last_cleanup > 300:
+                self._cleanup_expired()
+                self._last_cleanup = current_time
+        else:
+            self._last_cleanup = current_time
+        
+        # 創建消息鍵
+        message_key = self._create_message_key(sender, content)
+        
+        # 精確匹配檢查
+        if message_key in self.processed_messages:
+            age = current_time - self.processed_messages[message_key]
+            if age < self.expiry_seconds:
+                print(f"DUPLICATE EXACT: {sender} - {content[:40]}... (age: {age:.1f}s)")
+                return True
+            else:
+                # 過期了，移除
+                del self.processed_messages[message_key]
+        
+        # 相似性檢查（更嚴格）
+        clean_content = ' '.join(content.strip().split())
+        for existing_key, timestamp in list(self.processed_messages.items()):
+            age = current_time - timestamp
+            if age >= self.expiry_seconds:
+                continue
+                
+            try:
+                stored_sender, stored_content = existing_key.split(":", 1)
+                if sender.lower().strip() == stored_sender:
+                    # 計算相似度
+                    similarity = difflib.SequenceMatcher(None, clean_content, stored_content).ratio()
+                    if similarity >= 0.95:  # 95%相似度
+                        print(f"DUPLICATE SIMILAR: {sender} - {content[:40]}... (similarity: {similarity:.3f}, age: {age:.1f}s)")
+                        return True
+            except ValueError:
+                continue
+        
+        # 記錄新消息
+        self.processed_messages[message_key] = current_time
+        print(f"NEW MESSAGE RECORDED: {sender} - {content[:40]}...")
+        
+        # 異步保存（不阻塞主流程）
+        self._save_to_storage()
+        
+        return False
+    
     def clear_all(self):
-        """Clear all recorded messages (for F7/F8 functionality)."""
+        """清空所有記錄"""
-        count = len(self.processed_messages)
         self.processed_messages.clear()
-        if count > 0: # Log only if something was cleared
+        self._save_to_storage(force=True)
-            print(f"Deduplicator: Cleared all {count} message records.")
+        print("All dedup records cleared and persisted")
-        return count
+    
+    def get_stats(self):
+        """獲取統計信息"""
+        current_time = time.time()
+        active_count = sum(1 for timestamp in self.processed_messages.values() 
+                          if current_time - timestamp < self.expiry_seconds)
+        
+        return {
+            'total_records': len(self.processed_messages),
+            'active_records': active_count,
+            'oldest_record_age': min([current_time - t for t in self.processed_messages.values()]) if self.processed_messages else 0
+        }
+
+# 診斷工具：狀態重置檢測器
+class StateResetDetector:
+    def __init__(self, log_file="state_resets.log"):
+        self.log_file = log_file
+        self.start_time = time.time()
+        self.reset_count = 0
+        
+    def log_reset(self, reset_type, context=""):
+        """記錄狀態重置事件"""
+        self.reset_count += 1
+        timestamp = time.time()
+        
+        log_entry = f"{timestamp:.3f}: RESET #{self.reset_count} - {reset_type} - {context}\n"
+        
+        try:
+            with open(self.log_file, 'a', encoding='utf-8') as f:
+                f.write(log_entry)
+        except:
+            pass
+        
+        print(f"STATE RESET DETECTED: {reset_type} - {context}")
+    
+    def check_object_identity(self, obj, obj_name):
+        """檢查對象是否被重新創建"""
+        obj_id = id(obj)
+        attr_name = f"_{obj_name}_last_id"
+        
+        if hasattr(self, attr_name):
+            last_id = getattr(self, attr_name)
+            if last_id != obj_id:
+                self.log_reset("OBJECT_RECREATED", f"{obj_name} object was recreated")
+        
+        setattr(self, attr_name, obj_id)
 
 # --- Global Pause Flag ---
 # Using a simple mutable object (list) for thread-safe-like access without explicit lock
@@ -1707,12 +1822,13 @@ def perform_state_cleanup(detector: DetectionModule, interactor: InteractionModu
 
 
 # --- UI Monitoring Loop Function (To be run in a separate thread) ---
-def run_ui_monitoring_loop(trigger_queue: queue.Queue, command_queue: queue.Queue, deduplicator: 'MessageDeduplication'):
+def run_ui_monitoring_loop_enhanced(trigger_queue: queue.Queue, command_queue: queue.Queue, deduplicator: 'RobustMessageDeduplication', state_monitor: 'StateResetDetector'):
     """
     Continuously monitors the UI, detects triggers, performs interactions,
     puts trigger data into trigger_queue, and processes commands from command_queue.
+    Includes state monitoring and robust deduplication.
     """
-    print("\n--- Starting UI Monitoring Loop (Thread) ---")
+    print("\n--- Starting Enhanced UI Monitoring Loop (Thread) ---")
 
     # --- 初始化氣泡圖像去重系統（新增） ---
     bubble_deduplicator = SimpleBubbleDeduplication(
@@ -1792,8 +1908,18 @@ def run_ui_monitoring_loop(trigger_queue: queue.Queue, command_queue: queue.Queu
     main_screen_click_counter = 0 # Counter for consecutive main screen clicks
 
     loop_counter = 0 # Add loop counter for debugging
+    
     while True:
         loop_counter += 1
+        
+        # 每100次循環檢查一次對象狀態
+        if loop_counter % 100 == 0:
+            state_monitor.check_object_identity(deduplicator, "deduplicator")
+            
+            # 輸出統計信息
+            stats = deduplicator.get_stats()
+            print(f"Dedup Stats: {stats['active_records']} active records (total: {stats['total_records']})")
+        
         # print(f"\n--- UI Loop Iteration #{loop_counter} ---") # DEBUG REMOVED
 
         # --- Process ALL Pending Commands First ---
@@ -1853,10 +1979,8 @@ def run_ui_monitoring_loop(trigger_queue: queue.Queue, command_queue: queue.Queu
 
                     print("UI Thread: Resuming monitoring internally after restart wait.")
                     monitoring_paused_flag[0] = False
-                    # Clear state to ensure fresh detection after restart
+                    # 删除 recent_texts.clear() 和 last_processed_bubble_info = None
-                    recent_texts.clear()
+                    print("UI Thread: Monitoring resumed after restart. Duplicate detection state preserved.")
-                    last_processed_bubble_info = None
-                    print("UI Thread: Monitoring resumed and state reset after restart.")
                     # --- End Internal Sequence ---
 
                 elif action == 'clear_history': # Added for F7
@@ -2225,7 +2349,7 @@ def run_ui_monitoring_loop(trigger_queue: queue.Queue, command_queue: queue.Queu
                     # This is the new central point for deduplication and recent_texts logic
                     if sender_name and bubble_text: # Ensure both are valid before deduplication
                         if deduplicator.is_duplicate(sender_name, bubble_text):
-                            print(f"UI Thread: Skipping duplicate message via Deduplicator: {sender_name} - {bubble_text[:30]}...")
+                            print(f"UI Thread: Message blocked by robust deduplication: {sender_name} - {bubble_text[:30]}...")
                             # Cleanup UI state as interaction might have occurred during sender_name retrieval
                             perform_state_cleanup(detector, interactor)
                             continue  # Skip this bubble