diff --git a/.gitignore b/.gitignore index 01217dc8..a16c486e 100644 --- a/.gitignore +++ b/.gitignore @@ -53,6 +53,7 @@ memory/L4_raw_sessions/* # ljqCtrl related tools !memory/ljqCtrl.py !memory/ljqCtrl_sop.md +!memory/computer_use.md # procmem_scanner related tools !memory/procmem_scanner.py diff --git a/assets/global_mem_insight_template.txt b/assets/global_mem_insight_template.txt index 905ea7fa..3385393c 100644 --- a/assets/global_mem_insight_template.txt +++ b/assets/global_mem_insight_template.txt @@ -7,7 +7,8 @@ L3: memory_cleanup_sop(记忆整理) | skill_search | ui_detect.py | ocr_utils.p L4: L4_raw_sessions/ 历史会话 浏览器特殊操作: tmwebdriver_sop(文件上传/图搜/PDF blob/物理坐标/HttpOnly Cookie/autofill突破/跨域iframe/CDP/跨tab) -键鼠: ljqCtrl_sop(禁pyautogui/先activate) 截图/视觉: ocr/vision_sop | 禁全屏截图,优先窗口 +键鼠: ljqCtrl_sop(禁pyautogui/先activate) +UI操作/截图/视觉: computer_use.md | vision_sop | 窗口截图必用ljqCtrl 定时:scheduled_task_sop | 自主:autonomous_operation_sop | watchdog/反射:agentmain --reflect 手机:adb_ui.py diff --git a/assets/global_mem_insight_template_en.txt b/assets/global_mem_insight_template_en.txt index bcab25db..0ab46ea0 100644 --- a/assets/global_mem_insight_template_en.txt +++ b/assets/global_mem_insight_template_en.txt @@ -7,7 +7,8 @@ L3: memory_cleanup_sop(memory cleanup) | skill_search | ui_detect.py | ocr_utils L4: L4_raw_sessions/ historical sessions Browser special ops: tmwebdriver_sop(file upload/image search/PDF blob/physical coords/HttpOnly Cookie/autofill bypass/cross-origin iframe/CDP/cross-tab) -Keyboard & Mouse: ljqCtrl_sop(no pyautogui/activate first) Screenshot/Vision: ocr/vision_sop | No fullscreen capture, prefer window +Keyboard/Mouse: ljqCtrl_sop(no pyautogui/activate first) +UI/Screenshot/Vision: computer_use.md | vision_sop | window screenshot use ljqCtrl Scheduling: scheduled_task_sop | Autonomous: autonomous_operation_sop | watchdog/reflect: agentmain --reflect Mobile: adb_ui.py diff --git a/memory/computer_use.md b/memory/computer_use.md new file mode 100644 index 00000000..0c2dd095 --- /dev/null +++ b/memory/computer_use.md @@ -0,0 +1,22 @@ +# computer_use + +L3 memory: ljqCtrl.py ljqCtrl_sop.md ui_detect.py + +## 1. 基础规则 +枚举窗口总是可用:GUI任务先用 win32gui 枚举标题/类名/rect,确定目标窗口、前台状态和客户区原点后再操作。 +优先使用 Python UIA;若UIA可用,探测和操作均可使用UIA;游戏不使用UIA; +一旦 UIA 对该窗口无效,立刻改用 ui_detect 定位; +ui_detect 不足时才用 vision,vision 只用于语义理解、确认界面状态和辅助判断目标,不可信其坐标,不要直接按 vision 坐标点击。 +Windows 下窗口截图和操作使用 ljqCtrl:严禁 pyautogui;记得先 Activate 到前台; +巨坑DPI:一律使用物理坐标; +ui_detect 的 bbox 是截图内坐标,点击前必须用 `ClientToScreen(hwnd,(0,0))/dpi_scale + bbox中心` 转屏幕物理坐标 +坐标转换禁用 `GetWindowRect` 或 DWM 窗口矩形直接加截图坐标(含标题栏/边框/阴影会错位) +ljqCtrl.Click 后会返回像素/前台变化,0% 或近 0% 变化立即停下诊断,禁止盲目重试。 +ljqCtrl 失效或目标为网络游戏时,必须使用硬件键鼠 Xbananakb / Arduino Leonardo(如有) +网络游戏除非用户明确允许,严禁普通键鼠事件,必须硬件执行。 + +## 2. GUI操作节奏建议 +进入新界面时,建议先只探测不操作:枚举窗口 + UIA + ljqCtrl截图 + ui_detect,读完实际输出再决定下一步 +明确一个操作后,可以在同一轮执行该动作,短暂等待,再立刻枚举窗口 + 截图/ui_detect 验证新状态;不要在未知状态下把多步决策写进大脚本 +尽量不要预测关键词筛候选,应看 detect 输出、坐标、层级和上下文判断 +若确定UIA可用则少用ui_detect/ljqCtrl;若UIA不可用,则后续不用UIA diff --git a/memory/ui_detect.py b/memory/ui_detect.py index 7d679a1c..e0ff9205 100644 --- a/memory/ui_detect.py +++ b/memory/ui_detect.py @@ -11,9 +11,9 @@ UI元素检测 - YOLO + RapidOCR 依赖: ultralytics, rapidocr-onnxruntime, pillow, numpy """ from pathlib import Path -from ultralytics import YOLO from PIL import Image, ImageDraw import numpy as np +import json, urllib.request, subprocess, sys, time print('[UI DETECT] 截图分析后必须使用物理坐标,ljqCtrl也使用物理坐标!') @@ -22,13 +22,17 @@ DEFAULT_MODEL = str(Path(__file__).resolve().parent.parent / 'temp' / 'weights' try: from rapidocr_onnxruntime import RapidOCR _ocr = RapidOCR() -except ImportError: - _ocr = None +except ImportError: _ocr = None -def _yolo(image_path, model_path=None, conf=0.25): - """YOLO检测 → list of [x1,y1,x2,y2,conf]""" - model = YOLO(model_path or DEFAULT_MODEL) - res = model(image_path, conf=conf, verbose=False) +_YOLO = None +_YOLO_PORT = 31876 + +def _yolo_local(image_path, conf=0.25): + global _YOLO + if _YOLO is None: + from ultralytics import YOLO + _YOLO = YOLO(DEFAULT_MODEL) + res = _YOLO(image_path, conf=conf, verbose=False) boxes = [] for r in res: for b in r.boxes: @@ -36,6 +40,25 @@ def _yolo(image_path, model_path=None, conf=0.25): boxes.append([x1, y1, x2, y2, float(b.conf[0])]) return boxes + +def _ping_yolo_daemon(): + try: return urllib.request.urlopen(f'http://127.0.0.1:{_YOLO_PORT}/ping', timeout=0.1).read() == b'ui_detect_yolo' + except Exception: return False + +def _yolo(image_path, conf=0.25): + """YOLO检测 → list of [x1,y1,x2,y2,conf];默认模型走跨进程daemon cache,失败回退本地""" + if not _ping_yolo_daemon(): + kw = {'creationflags': getattr(subprocess, 'CREATE_NO_WINDOW', 0)} if sys.platform == 'win32' else {} + subprocess.Popen([sys.executable, __file__, '--yolo-daemon'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, **kw) + for _ in range(15): + if _ping_yolo_daemon(): break + time.sleep(0.5) + try: + data = json.dumps({'path': str(image_path), 'conf': conf}).encode('utf-8') + req = urllib.request.Request(f'http://127.0.0.1:{_YOLO_PORT}/yolo', data=data, headers={'Content-Type': 'application/json'}) + return json.loads(urllib.request.urlopen(req, timeout=3).read().decode('utf-8'))['boxes'] + except Exception: return _yolo_local(image_path, conf) + def _ocr_full(image_path): """全图OCR → list of [x1,y1,x2,y2,text,conf]""" if not _ocr: return [] @@ -87,7 +110,7 @@ def _iou(a, b): area_b = (b[2]-b[0]) * (b[3]-b[1]) return inter / area_b if area_b > 0 else 0 -def detect(image_path, mode='match', model_path=None, conf=0.25, iou_thresh=0.5): +def detect(image_path, mode='match', conf=0.25, iou_thresh=0.5): """ 统一检测入口,返回元素列表: [{'bbox':[x1,y1,x2,y2], 'type':'icon'|'text', 'label':str|None, 'confidence':float}] @@ -102,7 +125,7 @@ def detect(image_path, mode='match', model_path=None, conf=0.25, iou_thresh=0.5) image_path = tmp.name img = Image.open(image_path) - yolo_boxes = _yolo(image_path, model_path, conf) + yolo_boxes = _yolo(image_path, conf) elements = [] if mode == 'crop': @@ -133,7 +156,7 @@ def detect(image_path, mode='match', model_path=None, conf=0.25, iou_thresh=0.5) #if [x for x in elements if x['label'] is None]: print('[TIPS] crop grid + VLM to identify target no text icon if needed') return elements -def visualize(image_path, elements, output_path=None): +def visualize_for_debug(image_path, elements, output_path=None): """调试用: 可视化元素列表""" from PIL import ImageFont img = Image.open(image_path) @@ -151,4 +174,26 @@ def visualize(image_path, elements, output_path=None): if output_path: img.save(output_path) return img +def _serve_yolo_daemon(): + from http.server import BaseHTTPRequestHandler, HTTPServer + class H(BaseHTTPRequestHandler): + def log_message(self, *args): pass + def handle_one_request(self): self.server.last=time.time(); return super().handle_one_request() + def do_GET(self): + if self.path == '/ping': + self.send_response(200); self.end_headers(); self.wfile.write(b'ui_detect_yolo') + else: + self.send_response(404); self.end_headers() + def do_POST(self): + try: + d = json.loads(self.rfile.read(int(self.headers.get('Content-Length', 0)))) + body = json.dumps({'boxes': _yolo_local(d['path'], d.get('conf', 0.25))}).encode('utf-8') + self.send_response(200); self.end_headers(); self.wfile.write(body) + except Exception as e: + body = json.dumps({'error': repr(e)}).encode('utf-8') + self.send_response(500); self.end_headers(); self.wfile.write(body) + s=HTTPServer(('127.0.0.1', _YOLO_PORT), H); s.timeout=60; s.last=time.time() + while time.time()-s.last < 3600: s.handle_request() +if __name__ == '__main__' and '--yolo-daemon' in sys.argv: + _serve_yolo_daemon()