diff --git a/.gitignore b/.gitignore index 8c1c8816..47a6807f 100644 --- a/.gitignore +++ b/.gitignore @@ -55,6 +55,7 @@ memory/L4_raw_sessions/* !memory/ljqCtrl_sop.md !memory/macljqCtrl.py !memory/computer_use.md +!memory/ljqCtrlBg.py # procmem_scanner related tools !memory/procmem_scanner.py diff --git a/memory/computer_use.md b/memory/computer_use.md index a6efa700..3a4a34f8 100644 --- a/memory/computer_use.md +++ b/memory/computer_use.md @@ -1,6 +1,12 @@ # computer_use -相关L3 memory: **ui_detect.py** ljqCtrl.py ljqCtrl_sop.md +相关L3 memory: **ui_detect.py** ljqCtrl.py/ljqCtrlBg.py ljqCtrl_sop.md + +## 0. GUI操作节奏建议 +进入新界面时，建议先只探测不操作：枚举窗口 + UIA + ljqCtrl截图 + ui_detect，读完实际输出再决定下一步 +明确一个操作后，可以在同一轮执行该动作，短暂等待，再立刻枚举窗口 + 截图/ui_detect 验证新状态；不要在未知状态下把多步决策写进大脚本 +尽量不要预测关键词筛候选，应看 detect 输出、坐标、层级和上下文判断 +若确定UIA可用则少用ui_detect/ljqCtrl；若UIA不可用，则后续不用UIA ## 1. 基础规则 ### 探测/定位四工具（按优先级降级，前者无效才用后者） @@ -11,25 +17,25 @@ | 2 ui_detect.py（配合ljqCtrl截图） | 1无效时才用 | 截图视觉检测控件，返回 bbox+OCR 文本 | bbox 是截图内坐标需转屏幕物理坐标 | | 3 vision(VLM) | 2仍不足时才用 | 仅语义理解、确认界面状态、辅助判断目标 | 不可信其坐标 | -Windows 下窗口截图和操作使用 ljqCtrl：严禁 pyautogui；记得先 Activate 到前台； +Windows 下窗口截图和操作使用 ljqCtrl：严禁 pyautogui；记得先 Activate 到前台（除非用户明确要求后台操作或后台操作失效） ui_detect附送OCR，不要单独使用OCR -坑1-DPI：必须先import ljqCtrl，之后一律使用物理坐标；坑2-遮盖/失焦：混乱先枚举窗口确认前台； +用PIL传输图像，或者用统一1.png存储截图，不要创建大量截图文件 + ui_detect 的 bbox 是截图内坐标，点击前必须用 `ClientToScreen(hwnd,(0,0))/dpi_scale + bbox中心` 转屏幕物理坐标 坐标转换禁用 `GetWindowRect` 或 DWM 窗口矩形直接加截图坐标（含标题栏/边框/阴影会错位） ljqCtrl.Click 后会返回像素/前台变化，0% 或近 0% 变化立即停下诊断，禁止盲目重试。 ljqCtrl 失效或目标为网络游戏时，必须使用硬件键鼠 Xbananakb / Arduino Leonardo（如有） 网络游戏除非用户明确允许，严禁普通键鼠事件，必须硬件执行。 -## 2. GUI操作节奏建议 -进入新界面时，建议先只探测不操作：枚举窗口 + UIA + ljqCtrl截图 + ui_detect，读完实际输出再决定下一步 -明确一个操作后，可以在同一轮执行该动作，短暂等待，再立刻枚举窗口 + 截图/ui_detect 验证新状态；不要在未知状态下把多步决策写进大脚本 -尽量不要预测关键词筛候选，应看 detect 输出、坐标、层级和上下文判断 -若确定UIA可用则少用ui_detect/ljqCtrl；若UIA不可用，则后续不用UIA - 临时截图/可视化文件用后清理，或固定文件名覆盖，避免堆积。 ui_detect 可跨端复用；手机端沿用本原则时，UIA 换成 ui dump/adb_ui，ljqCtrl 控制换成 adb -## 3. macOS 平台 +### 重要必坑 +坑1-遮盖/失焦：混乱时枚举窗口确认前台； +坑2-DPI：必须先import ljqCtrl，之后一律使用物理坐标； + + +## macOS 平台 macOS 定位链与 §1 一致，工具映射如下： - 控制层：`import macljqCtrl as ljqCtrl`（替代 Windows ljqCtrl） - 窗口枚举：`ListWindows()` → 返回 id/app/title/bbox/pid（替代 win32gui） diff --git a/memory/ljqCtrlBg.py b/memory/ljqCtrlBg.py new file mode 100644 index 00000000..3f75a71f --- /dev/null +++ b/memory/ljqCtrlBg.py @@ -0,0 +1,200 @@ +"""ljqCtrlBg: concise background window control in client pixels. + +Never activates a window, moves the cursor, or injects global input. Posted +mouse/key messages are best-effort; always verify visible effects by screenshot. +""" +from __future__ import annotations +import ctypes, time +from typing import Any, NamedTuple, Optional, Sequence, Union +import win32con, win32gui, win32ui +from PIL import Image, ImageChops +try: + ctypes.windll.user32.SetProcessDPIAware() +except Exception: pass + +HwndLike = Union[int, str] +SMTO_SAFE = win32con.SMTO_BLOCK | win32con.SMTO_ABORTIFHUNG +CWP_SKIP = win32con.CWP_SKIPINVISIBLE | win32con.CWP_SKIPDISABLED | win32con.CWP_SKIPTRANSPARENT +MOUSE = {"left": (win32con.WM_LBUTTONDOWN, win32con.WM_LBUTTONUP, win32con.MK_LBUTTON), + "right": (win32con.WM_RBUTTONDOWN, win32con.WM_RBUTTONUP, win32con.MK_RBUTTON), + "middle": (win32con.WM_MBUTTONDOWN, win32con.WM_MBUTTONUP, win32con.MK_MBUTTON)} +KEYS = {"backspace": 8, "tab": 9, "enter": 13, "return": 13, "shift": 16, "ctrl": 17, "control": 17, + "alt": 18, "esc": 27, "escape": 27, "space": 32, "pageup": 33, "pagedown": 34, "end": 35, + "home": 36, "left": 37, "up": 38, "right": 39, "down": 40, "delete": 46, "del": 46} + +class CaptureResult(NamedTuple): + image: Image.Image + hwnd: int + backend: str + client_origin: tuple[int, int] + client_size: tuple[int, int] + size = property(lambda self: self.image.size) + origin_screen_phys = property(lambda self: self.client_origin) + client_size_phys = property(lambda self: self.client_size) + +def ListWindows(visible_only: bool = True) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + def each(hwnd: int, _: Any) -> bool: + title, vis = win32gui.GetWindowText(hwnd), win32gui.IsWindowVisible(hwnd); rect = tuple(map(int, win32gui.GetWindowRect(hwnd))) + if (vis or not visible_only) and (title or not visible_only): + rows.append({"hwnd": int(hwnd), "title": title, "class": win32gui.GetClassName(hwnd), "rect": rect, "visible": bool(vis)}) + return True + win32gui.EnumWindows(each, None); return rows + +def FindWindow(name: str, exact: bool = False, class_name: Optional[str] = None, visible_only: bool = True) -> int: + needle = str(name).lower() + for row in ListWindows(visible_only): + title = row["title"] or ""; ok = (title == name) if exact else (needle in title.lower()) + if ok and (class_name is None or row["class"] == class_name): return int(row["hwnd"]) + raise RuntimeError(f"window not found: {name!r}") + +def ResolveHwnd(hwnd_or_name: HwndLike) -> int: + hwnd = int(hwnd_or_name) if isinstance(hwnd_or_name, int) else FindWindow(str(hwnd_or_name)) + if not win32gui.IsWindow(hwnd): raise RuntimeError(f"invalid hwnd: {hwnd!r}") + return hwnd + +def GetWRect(hwnd_or_name: HwndLike) -> tuple[int, int, int, int]: + return tuple(map(int, win32gui.GetWindowRect(ResolveHwnd(hwnd_or_name)))) + +def ClientSize(hwnd_or_name: HwndLike) -> tuple[int, int]: + l, t, r, b = win32gui.GetClientRect(ResolveHwnd(hwnd_or_name)); return int(r - l), int(b - t) + +def ClientOrigin(hwnd_or_name: HwndLike) -> tuple[int, int]: + return tuple(map(int, win32gui.ClientToScreen(ResolveHwnd(hwnd_or_name), (0, 0)))) + +def ClientRectScreen(hwnd_or_name: HwndLike) -> tuple[int, int, int, int]: + x, y = ClientOrigin(hwnd_or_name); w, h = ClientSize(hwnd_or_name); return x, y, x + w, y + h + +def ScreenToClient(hwnd_or_name: HwndLike, x: int, y: int) -> tuple[int, int]: + return tuple(map(int, win32gui.ScreenToClient(ResolveHwnd(hwnd_or_name), (int(x), int(y))))) + +def ClientToScreen(hwnd_or_name: HwndLike, x: int, y: int) -> tuple[int, int]: + return tuple(map(int, win32gui.ClientToScreen(ResolveHwnd(hwnd_or_name), (int(x), int(y))))) + +def ChildAt(hwnd_or_name: HwndLike, x: int, y: int, coords: str = "client", deep: bool = True) -> tuple[int, int, int]: + if coords not in {"client", "screen"}: raise ValueError("coords must be 'client' or 'screen'") + root = ResolveHwnd(hwnd_or_name); sx, sy = ClientToScreen(root, x, y) if coords == "client" else (int(x), int(y)) + hwnd = root + while deep: + cx, cy = win32gui.ScreenToClient(hwnd, (sx, sy)) + child = win32gui.ChildWindowFromPointEx(hwnd, (cx, cy), CWP_SKIP) + if not child or child == hwnd: break + hwnd = child + cx, cy = win32gui.ScreenToClient(hwnd, (sx, sy)); return int(hwnd), int(cx), int(cy) + +def _crop_client(hwnd: int, image: Image.Image, size: tuple[int, int]) -> Image.Image: + if image.size == size: return image + wx, wy, _, _ = win32gui.GetWindowRect(hwnd); ox, oy = ClientOrigin(hwnd); w, h = size; dx, dy = ox - wx, oy - wy + if 0 <= dx and 0 <= dy and dx + w <= image.width and dy + h <= image.height: return image.crop((dx, dy, dx + w, dy + h)) + if image.width >= w and image.height >= h: return image.crop((0, 0, w, h)) + raise RuntimeError(f"capture frame {image.size} smaller than client area {size}") + +def _grab_wgc(hwnd: int, size: tuple[int, int], timeout: float) -> Image.Image: + from windows_capture import WindowsCapture # type: ignore + frames: list[Any] = []; errors: list[BaseException] = [] + cap = WindowsCapture(cursor_capture=False, draw_border=False, window_hwnd=hwnd) + @cap.event + def on_frame_arrived(frame: Any, control: Any) -> None: + try: frames.append(frame.frame_buffer.copy()) + except BaseException as exc: errors.append(exc) + finally: + try: control.stop() + except Exception: pass + @cap.event + def on_closed() -> None: pass + control = cap.start_free_threaded(); end = time.monotonic() + float(timeout) + while not frames and not errors and time.monotonic() < end: time.sleep(0.02) + if not frames: + try: control.stop() + except Exception: pass + if errors: raise RuntimeError("WGC callback failed") from errors[0] + raise TimeoutError(f"WGC did not produce a frame within {timeout:.1f}s") + arr = frames[0] + if getattr(arr, "ndim", 0) != 3 or arr.shape[2] < 3: raise RuntimeError(f"bad WGC frame shape: {getattr(arr, 'shape', None)}") + return _crop_client(hwnd, Image.fromarray(arr[:, :, :3][:, :, ::-1]).copy(), size) + +def _grab_printwindow(hwnd: int, size: tuple[int, int]) -> tuple[Image.Image, bool]: + w, h = size; hdc = win32gui.GetWindowDC(hwnd); src = win32ui.CreateDCFromHandle(hdc) + mem = src.CreateCompatibleDC(); bmp = win32ui.CreateBitmap(); bmp.CreateCompatibleBitmap(src, w, h); old = mem.SelectObject(bmp) + try: + ok = bool(ctypes.windll.user32.PrintWindow(hwnd, mem.GetSafeHdc(), 1)) + info, bits = bmp.GetInfo(), bmp.GetBitmapBits(True) + return Image.frombuffer("RGB", (info["bmWidth"], info["bmHeight"]), bits, "raw", "BGRX", 0, 1).copy(), ok + finally: + mem.SelectObject(old); win32gui.DeleteObject(bmp.GetHandle()); mem.DeleteDC(); src.DeleteDC(); win32gui.ReleaseDC(hwnd, hdc) + +def GrabWindowBg(hwnd_or_name: HwndLike, backend: str = "auto", timeout: float = 3.0) -> CaptureResult: + hwnd = ResolveHwnd(hwnd_or_name); size = ClientSize(hwnd); mode = backend.lower(); wgc_error = "" + if min(size) <= 0: raise RuntimeError(f"empty client area for hwnd={hwnd}") + if mode in {"auto", "wgc"}: + try: return CaptureResult(_grab_wgc(hwnd, size, timeout), hwnd, "wgc", ClientOrigin(hwnd), size) + except BaseException as exc: + if mode == "wgc": raise + wgc_error = f";wgc-error={type(exc).__name__}" + if mode in {"auto", "printwindow", "pw"}: + image, ok = _grab_printwindow(hwnd, size); label = "printwindow" if ok else "printwindow-best-effort" + return CaptureResult(image, hwnd, label + wgc_error, ClientOrigin(hwnd), size) + raise ValueError("backend must be 'auto', 'wgc', or 'printwindow'") + +def GrabClientBg(hwnd_or_name: HwndLike, **kwargs: Any) -> Image.Image: + return GrabWindowBg(hwnd_or_name, **kwargs).image + +def _lparam(x: int, y: int) -> int: return (int(x) & 0xFFFF) | ((int(y) & 0xFFFF) << 16) + +def _send(hwnd: int, msg: int, wp: int = 0, lp: int = 0, post: bool = True) -> None: + if post: win32gui.PostMessage(hwnd, msg, int(wp), int(lp)) + else: win32gui.SendMessageTimeout(hwnd, msg, int(wp), int(lp), SMTO_SAFE, 1000) + +def ClickBg(hwnd_or_name: HwndLike, x: int, y: int, button: str = "left", coords: str = "client", target_child: bool = True, post: bool = True, interval: float = 0.03, check: bool = True, r: int = 80, wait: float = 0.5) -> bool: + root = ResolveHwnd(hwnd_or_name); wins0 = {w["hwnd"]: (w["title"], w["class"]) for w in ListWindows(False)} if check else {}; cap1 = GrabWindowBg(root) if check else None + if button.lower() not in MOUSE: raise ValueError(f"unsupported button: {button!r}") + if target_child: hwnd, cx, cy = ChildAt(root, x, y, coords) + else: hwnd, cx, cy = root, *(ScreenToClient(root, x, y) if coords == "screen" else (int(x), int(y))) + down, up, mk = MOUSE[button.lower()]; lp = _lparam(cx, cy); _send(hwnd, win32con.WM_MOUSEMOVE, 0, lp, post); _send(hwnd, down, mk, lp, post) + if interval: time.sleep(float(interval)) + _send(hwnd, up, 0, lp, post) + if check: + time.sleep(float(wait)); wins1 = {w["hwnd"]: (w["title"], w["class"]) for w in ListWindows(False)}; new = {k: v for k, v in wins1.items() if k not in wins0}; gone = {k: v for k, v in wins0.items() if k not in wins1}; bbox = None + if win32gui.IsWindow(root): cap2 = GrabWindowBg(root); im1 = cap1.image.crop((max(0, x-r), max(0, y-r), min(cap1.size[0], x+r), min(cap1.size[1], y+r))); im2 = cap2.image.crop((max(0, x-r), max(0, y-r), min(cap2.size[0], x+r), min(cap2.size[1], y+r))); bbox = ImageChops.difference(im1, im2).getbbox() + print(f"[ClickBg check] changed={bool(bbox)} bbox={bbox} new={new} gone={gone}") + return True + +def Click(hwnd_or_name: HwndLike, x: int, y: int, **kwargs: Any) -> bool: return ClickBg(hwnd_or_name, x, y, **kwargs) + +def _vk(key: Union[str, int]) -> int: + if isinstance(key, int): return int(key) + s = str(key).strip(); low = s.lower() + if low in KEYS: return int(KEYS[low]) + if low.startswith("f") and low[1:].isdigit() and 1 <= int(low[1:]) <= 24: return win32con.VK_F1 + int(low[1:]) - 1 + if len(s) == 1: return int(ctypes.windll.user32.VkKeyScanW(ord(s)) & 0xFF) + raise ValueError(f"unknown key: {key!r}") + +def _key_lparam(vk: int, up: bool = False) -> int: + lp = 1 | (int(ctypes.windll.user32.MapVirtualKeyW(int(vk), 0)) << 16) + return lp | ((1 << 30) | (1 << 31) if up else 0) + +def PressBg(hwnd_or_name: HwndLike, key: Union[str, int], modifiers: Optional[Sequence[Union[str, int]]] = None, post: bool = True, interval: float = 0.02) -> bool: + hwnd = ResolveHwnd(hwnd_or_name) + if isinstance(key, str) and "+" in key and modifiers is None: + parts = [p.strip() for p in key.split("+") if p.strip()]; mods, main = [_vk(p) for p in parts[:-1]], _vk(parts[-1]) + else: mods, main = [_vk(m) for m in (modifiers or [])], _vk(key) + for vk in [*mods, main]: _send(hwnd, win32con.WM_KEYDOWN, vk, _key_lparam(vk), post) + if interval: time.sleep(float(interval)) + for vk in [main, *reversed(mods)]: _send(hwnd, win32con.WM_KEYUP, vk, _key_lparam(vk, True), post) + return True + +def Press(hwnd_or_name: HwndLike, key: Union[str, int], **kwargs: Any) -> bool: return PressBg(hwnd_or_name, key, **kwargs) + +def TypeTextBg(hwnd_or_name: HwndLike, text: str, interval: float = 0.0, post: bool = True) -> bool: + hwnd = ResolveHwnd(hwnd_or_name) + for ch in str(text): + _send(hwnd, win32con.WM_CHAR, ord(ch), 1, post) + if interval: time.sleep(float(interval)) + return True + +def SetTextBg(hwnd_or_name: HwndLike, text: str) -> bool: + win32gui.SendMessage(ResolveHwnd(hwnd_or_name), win32con.WM_SETTEXT, 0, str(text)); return True + +def GetTextBg(hwnd_or_name: HwndLike) -> str: return win32gui.GetWindowText(ResolveHwnd(hwnd_or_name)) + +if __name__ == "__main__": print(f"ljqCtrlBg ready; windows={len(ListWindows())}") diff --git a/memory/ui_detect.py b/memory/ui_detect.py index 148d3f8e..0ac5a934 100644 --- a/memory/ui_detect.py +++ b/memory/ui_detect.py @@ -19,7 +19,7 @@ from PIL import Image, ImageDraw import numpy as np import json, urllib.request, subprocess, sys, time -print('[UI DETECT] 截图分析后必须使用物理坐标，ljqCtrl也使用物理坐标！') +#print('[UI DETECT] 截图分析后必须使用物理坐标，ljqCtrl也使用物理坐标！') DEFAULT_MODEL = str(Path(__file__).resolve().parent.parent / 'temp' / 'weights' / 'icon_detect' / 'model.pt') @@ -158,6 +158,7 @@ def detect(image_path, mode='match', conf=0.25, iou_thresh=0.5): if i not in matched_ocr: elements.append({'bbox': [ox1,oy1,ox2,oy2], 'type': 'text', 'label': text, 'confidence': oc}) #if [x for x in elements if x['label'] is None]: print('[TIPS] crop grid + VLM to identify target no text icon if needed') + print('[TIPS] UI DETECT contains OCR, no need to run OCR again!') return elements def visualize_for_debug(image_path, elements, output_path=None):