Add computer_use SOP for UI operations

memory/computer_use.md

@@ -0,0 +1,22 @@
+# computer_use
+
+L3 memory: ljqCtrl.py ljqCtrl_sop.md ui_detect.py
+
+## 1. 基础规则
+枚举窗口总是可用：GUI任务先用 win32gui 枚举标题/类名/rect，确定目标窗口、前台状态和客户区原点后再操作。
+优先使用 Python UIA；若UIA可用，探测和操作均可使用UIA；游戏不使用UIA；
+一旦 UIA 对该窗口无效，立刻改用 ui_detect 定位；
+ui_detect 不足时才用 vision，vision 只用于语义理解、确认界面状态和辅助判断目标，不可信其坐标，不要直接按 vision 坐标点击。
+Windows 下窗口截图和操作使用 ljqCtrl：严禁 pyautogui；记得先 Activate 到前台；
+巨坑DPI：一律使用物理坐标；
+ui_detect 的 bbox 是截图内坐标，点击前必须用 `ClientToScreen(hwnd,(0,0))/dpi_scale + bbox中心` 转屏幕物理坐标
+坐标转换禁用 `GetWindowRect` 或 DWM 窗口矩形直接加截图坐标（含标题栏/边框/阴影会错位）
+ljqCtrl.Click 后会返回像素/前台变化，0% 或近 0% 变化立即停下诊断，禁止盲目重试。
+ljqCtrl 失效或目标为网络游戏时，必须使用硬件键鼠 Xbananakb / Arduino Leonardo（如有）
+网络游戏除非用户明确允许，严禁普通键鼠事件，必须硬件执行。
+
+## 2. GUI操作节奏建议
+进入新界面时，建议先只探测不操作：枚举窗口 + UIA + ljqCtrl截图 + ui_detect，读完实际输出再决定下一步
+明确一个操作后，可以在同一轮执行该动作，短暂等待，再立刻枚举窗口 + 截图/ui_detect 验证新状态；不要在未知状态下把多步决策写进大脚本
+尽量不要预测关键词筛候选，应看 detect 输出、坐标、层级和上下文判断
+若确定UIA可用则少用ui_detect/ljqCtrl；若UIA不可用，则后续不用UIA

memory/ui_detect.py

@@ -11,9 +11,9 @@ UI元素检测 - YOLO + RapidOCR
 依赖: ultralytics, rapidocr-onnxruntime, pillow, numpy
 """
 from pathlib import Path
-from ultralytics import YOLO
 from PIL import Image, ImageDraw
 import numpy as np
+import json, urllib.request, subprocess, sys, time
 
 print('[UI DETECT] 截图分析后必须使用物理坐标，ljqCtrl也使用物理坐标！')
 
@@ -22,13 +22,17 @@ DEFAULT_MODEL = str(Path(__file__).resolve().parent.parent / 'temp' / 'weights'
 try:
     from rapidocr_onnxruntime import RapidOCR
     _ocr = RapidOCR()
-except ImportError:
-    _ocr = None
+except ImportError: _ocr = None
 
-def _yolo(image_path, model_path=None, conf=0.25):
-    """YOLO检测 → list of [x1,y1,x2,y2,conf]"""
-    model = YOLO(model_path or DEFAULT_MODEL)
-    res = model(image_path, conf=conf, verbose=False)
+_YOLO = None
+_YOLO_PORT = 31876
+
+def _yolo_local(image_path, conf=0.25):
+    global _YOLO
+    if _YOLO is None:
+        from ultralytics import YOLO
+        _YOLO = YOLO(DEFAULT_MODEL)
+    res = _YOLO(image_path, conf=conf, verbose=False)
     boxes = []
     for r in res:
         for b in r.boxes:
@@ -36,6 +40,25 @@ def _yolo(image_path, model_path=None, conf=0.25):
             boxes.append([x1, y1, x2, y2, float(b.conf[0])])
     return boxes
 
+
+def _ping_yolo_daemon():
+    try: return urllib.request.urlopen(f'http://127.0.0.1:{_YOLO_PORT}/ping', timeout=0.1).read() == b'ui_detect_yolo'
+    except Exception: return False
+
+def _yolo(image_path, conf=0.25):
+    """YOLO检测 → list of [x1,y1,x2,y2,conf]；默认模型走跨进程daemon cache，失败回退本地"""
+    if not _ping_yolo_daemon():
+        kw = {'creationflags': getattr(subprocess, 'CREATE_NO_WINDOW', 0)} if sys.platform == 'win32' else {}
+        subprocess.Popen([sys.executable, __file__, '--yolo-daemon'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, **kw)
+        for _ in range(15):
+            if _ping_yolo_daemon(): break
+            time.sleep(0.5)
+    try:
+        data = json.dumps({'path': str(image_path), 'conf': conf}).encode('utf-8')
+        req = urllib.request.Request(f'http://127.0.0.1:{_YOLO_PORT}/yolo', data=data, headers={'Content-Type': 'application/json'})
+        return json.loads(urllib.request.urlopen(req, timeout=3).read().decode('utf-8'))['boxes']
+    except Exception: return _yolo_local(image_path, conf)
+
 def _ocr_full(image_path):
     """全图OCR → list of [x1,y1,x2,y2,text,conf]"""
     if not _ocr: return []
@@ -87,7 +110,7 @@ def _iou(a, b):
     area_b = (b[2]-b[0]) * (b[3]-b[1])
     return inter / area_b if area_b > 0 else 0
 
-def detect(image_path, mode='match', model_path=None, conf=0.25, iou_thresh=0.5):
+def detect(image_path, mode='match', conf=0.25, iou_thresh=0.5):
     """
     统一检测入口，返回元素列表:
     [{'bbox':[x1,y1,x2,y2], 'type':'icon'|'text', 'label':str|None, 'confidence':float}]
@@ -102,7 +125,7 @@ def detect(image_path, mode='match', model_path=None, conf=0.25, iou_thresh=0.5)
         image_path = tmp.name
     img = Image.open(image_path)
 
-    yolo_boxes = _yolo(image_path, model_path, conf)
+    yolo_boxes = _yolo(image_path, conf)
     elements = []
 
     if mode == 'crop':
@@ -133,7 +156,7 @@ def detect(image_path, mode='match', model_path=None, conf=0.25, iou_thresh=0.5)
     #if [x for x in elements if x['label'] is None]: print('[TIPS] crop grid + VLM to identify target no text icon if needed')
     return elements
 
-def visualize(image_path, elements, output_path=None):
+def visualize_for_debug(image_path, elements, output_path=None):
     """调试用: 可视化元素列表"""
     from PIL import ImageFont
     img = Image.open(image_path)
@@ -151,4 +174,26 @@ def visualize(image_path, elements, output_path=None):
     if output_path: img.save(output_path)
     return img
 
+def _serve_yolo_daemon():
+    from http.server import BaseHTTPRequestHandler, HTTPServer
+    class H(BaseHTTPRequestHandler):
+        def log_message(self, *args): pass
+        def handle_one_request(self): self.server.last=time.time(); return super().handle_one_request()
+        def do_GET(self):
+            if self.path == '/ping':
+                self.send_response(200); self.end_headers(); self.wfile.write(b'ui_detect_yolo')
+            else:
+                self.send_response(404); self.end_headers()
+        def do_POST(self):
+            try:
+                d = json.loads(self.rfile.read(int(self.headers.get('Content-Length', 0))))
+                body = json.dumps({'boxes': _yolo_local(d['path'], d.get('conf', 0.25))}).encode('utf-8')
+                self.send_response(200); self.end_headers(); self.wfile.write(body)
+            except Exception as e:
+                body = json.dumps({'error': repr(e)}).encode('utf-8')
+                self.send_response(500); self.end_headers(); self.wfile.write(body)
+    s=HTTPServer(('127.0.0.1', _YOLO_PORT), H); s.timeout=60; s.last=time.time()
+    while time.time()-s.last < 3600: s.handle_request()
 
+if __name__ == '__main__' and '--yolo-daemon' in sys.argv:
+    _serve_yolo_daemon()