ljqCtrl: add Activate/GetForeground/Click check; SOP精简39行; ui_detect物理坐标提醒
This commit is contained in:
Liang Jiaqing committed
1 parent
0a2f96348d
commit
c769c4cb1a
3 files changed
+44
-29
No files matched your search
+27
-6
@@ -10,10 +10,11 @@ ljqCtrl Quick Reference:
|
||||
- GrabWindow(hwnd) -> PIL Image: DPI-safe window screenshot (needs foreground)
|
||||
- GrabWindowBg(hwnd_or_name) -> PIL Image: WGC background capture (Win10+, pip install windows-capture)
|
||||
"""
|
||||
|
||||
import os, sys, time, random, math, win32api, win32con, ctypes
|
||||
import os, sys, time, random, math, win32api, win32con, win32gui, ctypes
|
||||
import numpy as np
|
||||
|
||||
print('[TIPS] always use physical coordinates!')
|
||||
|
||||
dpi_scale = 1
|
||||
try:
|
||||
from PIL import ImageGrab, Image, ImageEnhance, ImageFilter, ImageDraw
|
||||
@@ -49,16 +50,19 @@ def SetCursorPos(z):
|
||||
|
||||
def Click(x, y=None, check=True):
|
||||
if type(x) is type(tuple()): x, y = int(x[0]), int(x[1])
|
||||
if check: before = ScreenCapAt(x, y)
|
||||
if check: before, fg_before = ScreenCapAt(x, y), win32gui.GetForegroundWindow()
|
||||
SetCursorPos( (x, y) )
|
||||
MouseClick()
|
||||
if check:
|
||||
time.sleep(0.3)
|
||||
time.sleep(0.5)
|
||||
after = ScreenCapAt(x, y)
|
||||
b, a = np.array(before), np.array(after)
|
||||
diff = np.sum(np.any(b != a, axis=2))
|
||||
total = b.shape[0] * b.shape[1]
|
||||
print(f'[Click check] near 100px rect {diff}/{total} pixels changed ({diff/total*100:.1f}%)')
|
||||
fg_after = win32gui.GetForegroundWindow()
|
||||
fg_title = win32gui.GetWindowText(fg_after)
|
||||
fg_changed = fg_before != fg_after
|
||||
print(f'[Click check] {diff}/{total} px changed ({diff/total*100:.1f}%) | fg: "{fg_title}" {"⚠️CHANGED" if fg_changed else ""}')
|
||||
return after
|
||||
click = Click
|
||||
|
||||
@@ -76,8 +80,25 @@ press = Press
|
||||
VK_CODE = {'backspace':0x08, 'tab':0x09, 'clear':0x0C, 'enter':0x0D, 'shift':0x10, 'ctrl':0x11, 'alt':0x12, 'pause':0x13, 'caps_lock':0x14, 'esc':0x1B, 'escape':0x1B, 'space':0x20, 'page_up':0x21, 'page_down':0x22, 'end':0x23, 'home':0x24, 'left_arrow':0x25, 'up_arrow':0x26, 'right_arrow':0x27, 'down_arrow':0x28, 'select':0x29, 'print':0x2A, 'execute':0x2B, 'print_screen':0x2C, 'ins':0x2D, 'del':0x2E, 'help':0x2F, '0':0x30, '1':0x31, '2':0x32, '3':0x33, '4':0x34, '5':0x35, '6':0x36, '7':0x37, '8':0x38, '9':0x39, 'a':0x41, 'b':0x42, 'c':0x43, 'd':0x44, 'e':0x45, 'f':0x46, 'g':0x47, 'h':0x48, 'i':0x49, 'j':0x4A, 'k':0x4B, 'l':0x4C, 'm':0x4D, 'n':0x4E, 'o':0x4F, 'p':0x50, 'q':0x51, 'r':0x52, 's':0x53, 't':0x54, 'u':0x55, 'v':0x56, 'w':0x57, 'x':0x58, 'y':0x59, 'z':0x5A, 'numpad_0':0x60, 'numpad_1':0x61, 'numpad_2':0x62, 'numpad_3':0x63, 'numpad_4':0x64, 'numpad_5':0x65, 'numpad_6':0x66, 'numpad_7':0x67, 'numpad_8':0x68, 'numpad_9':0x69, 'multiply_key':0x6A, 'add_key':0x6B, 'separator_key':0x6C, 'subtract_key':0x6D, 'decimal_key':0x6E, 'divide_key':0x6F, 'F1':0x70, 'F2':0x71, 'F3':0x72, 'F4':0x73, 'F5':0x74, 'F6':0x75, 'F7':0x76, 'F8':0x77, 'F9':0x78, 'F10':0x79, 'F11':0x7A, 'F12':0x7B, 'F13':0x7C, 'F14':0x7D, 'F15':0x7E, 'F16':0x7F, 'F17':0x80, 'F18':0x81, 'F19':0x82, 'F20':0x83, 'F21':0x84, 'F22':0x85, 'F23':0x86, 'F24':0x87, 'num_lock':0x90, 'scroll_lock':0x91, 'left_shift':0xA0, 'right_shift ':0xA1, 'left_control':0xA2, 'right_control':0xA3, 'left_menu':0xA4, 'right_menu':0xA5, 'browser_back':0xA6, 'browser_forward':0xA7, 'browser_refresh':0xA8, 'browser_stop':0xA9, 'browser_search':0xAA, 'browser_favorites':0xAB, 'browser_start_and_home':0xAC, 'volume_mute':0xAD, 'volume_Down':0xAE, 'volume_up':0xAF, 'next_track':0xB0, 'previous_track':0xB1, 'stop_media':0xB2, 'play/pause_media':0xB3, 'start_mail':0xB4, 'select_media':0xB5, 'start_application_1':0xB6, 'start_application_2':0xB7, 'attn_key':0xF6, 'crsel_key':0xF7, 'exsel_key':0xF8, 'play_key':0xFA, 'zoom_key':0xFB, 'clear_key':0xFE, '+':0xBB, ',':0xBC, '-':0xBD, '.':0xBE, '/':0xBF, '`':0xC0, ';':0xBA, '[':0xDB, '\\':0xDC, ']':0xDD, "'":0xDE}
|
||||
VK_CODE = {k.lower():v for k,v in VK_CODE.items()}
|
||||
|
||||
def Activate(hwnd):
|
||||
"""稳定切换前台窗口。绕过Windows前台锁限制。"""
|
||||
# 如果窗口最小化先恢复
|
||||
if ctypes.windll.user32.IsIconic(hwnd):
|
||||
ctypes.windll.user32.ShowWindow(hwnd, 9) # SW_RESTORE
|
||||
# 发假Alt-up骗过前台锁
|
||||
ctypes.windll.user32.keybd_event(0x12, 0, 2, 0) # VK_MENU up
|
||||
time.sleep(0.02)
|
||||
try:
|
||||
win32gui.SetForegroundWindow(hwnd)
|
||||
except Exception:
|
||||
# fallback: BringWindowToTop + SetFocus
|
||||
ctypes.windll.user32.BringWindowToTop(hwnd)
|
||||
ctypes.windll.user32.SetFocus(hwnd)
|
||||
time.sleep(0.15)
|
||||
activate = Activate
|
||||
|
||||
def GrabWindow(hwnd):
|
||||
import win32gui; win32gui.SetForegroundWindow(hwnd); time.sleep(0.3)
|
||||
Activate(hwnd); time.sleep(0.25)
|
||||
bbox = tuple(int(v / dpi_scale) for v in win32gui.GetWindowRect(hwnd))
|
||||
return ImageGrab.grab(bbox)
|
||||
|
||||
|
||||
+14
-22
@@ -1,47 +1,39 @@
|
||||
# ljqCtrl 使用与坐标转换 SOP
|
||||
|
||||
> **must call update working ckp**：`ljqCtrl一律使用物理坐标｜禁pyautogui｜操作前先gw激活窗口`
|
||||
> **must call update working ckp**：`一律使用物理坐标｜禁pyautogui｜操作前先激活窗口`
|
||||
|
||||
## 0. API 快速参考 (Signatures)
|
||||
- `ljqCtrl.dpi_scale`: float (缩放系数 = 逻辑宽度 / 物理宽度)
|
||||
- `ljqCtrl.SetCursorPos(z)`: 移动鼠标到逻辑坐标 z=(x, y)
|
||||
- `ljqCtrl.Click(x, y=None)`: 模拟点击。支持 `Click((x, y))` 或 `Click(x, y)`
|
||||
- `ljqCtrl.Press(cmd, staytime=0)`: 模拟按键。如 `Press('ctrl+c')`
|
||||
- `ljqCtrl.FindBlock(fn, wrect=None, threshold=0.8)`: 找图。返回 `((center_x, center_y), is_found)`
|
||||
- `ljqCtrl.GrabWindowBg(hwnd_or_name, timeout=5)`: WGC后台截图(Win10+, pip install windows-capture), 传hwnd(int)或窗口标题子串(str), 返回PIL Image
|
||||
- `ljqCtrl.GrabWindow(hwnd_or_name)`: 前台截图(先Activate), 传hwnd(int)或窗口标题子串(str), 返回PIL Image
|
||||
- `ljqCtrl.GrabWindowBg(hwnd_or_name, timeout=5)`: WGC后台截图(Win10+)
|
||||
- `ljqCtrl.MouseDClick(staytime=0.05)`: 鼠标双击
|
||||
|
||||
## 1. 环境载入
|
||||
必须先将 `../memory` 加入路径，才能导入工具模块：
|
||||
```python
|
||||
import sys, os, pygetwindow as gw
|
||||
sys.path.append("../memory")
|
||||
import ljqCtrl
|
||||
```
|
||||
|
||||
## 2. 核心：High-DPI 物理坐标换算
|
||||
`ljqCtrl` 的 `Click/MoveTo` 接口接收的是**物理像素坐标**。
|
||||
当使用 `pygetwindow` 等工具获取窗口位置（逻辑坐标）时，必须除以缩放系数。
|
||||
当使用 `pygetwindow` 等其他工具获取窗口位置（逻辑坐标）时，必须除以缩放系数。
|
||||
|
||||
- **换算公式**：`物理坐标 = 逻辑坐标 / ljqCtrl.dpi_scale`
|
||||
- **注意**：3840 (4K) 仅为当前开发机示例，实际物理边界由系统环境决定，代码应始终通过 `dpi_scale` 动态计算。
|
||||
|
||||
## 3. 窗口操作与点击流程
|
||||
1. **激活窗口**：使用 `gw.getWindowsWithTitle('标题')` 获取窗口，执行 `restore()` 和 `activate()`。
|
||||
2. **坐标计算**：
|
||||
|
||||
## 3. 截图bbox → 屏幕物理坐标（核心公式）
|
||||
```python
|
||||
win = gw.getWindowsWithTitle('微信')[0]
|
||||
# 计算窗口内某个点的逻辑坐标 (lx, ly)
|
||||
# 转换为物理坐标并点击
|
||||
px, py = lx / ljqCtrl.dpi_scale, ly / ljqCtrl.dpi_scale
|
||||
ljqCtrl.Click(px, py)
|
||||
# ui_detect获取的都是物理坐标
|
||||
# ClientToScreen拿客户区原点(逻辑) → 除dpi_scale得物理偏移
|
||||
cx, cy = win32gui.ClientToScreen(hwnd, (0, 0))
|
||||
ox, oy = int(cx / ljqCtrl.dpi_scale), int(cy / ljqCtrl.dpi_scale)
|
||||
ljqCtrl.Click(ox + (bbox[0]+bbox[2])//2, oy + (bbox[1]+bbox[3])//2)
|
||||
```
|
||||
禁止全屏ImageGrab（必须针对窗口），所有逻辑坐标都要转物理。
|
||||
|
||||
## 4. 避坑指南
|
||||
- **⚠️ 一律使用物理坐标**：传给 ljqCtrl.Click/SetCursorPos 的坐标必须是物理坐标（=截图像素坐标）。从 pygetwindow 获取的逻辑坐标需先 `/ dpi_scale` 转换。禁止传入逻辑坐标。
|
||||
- **⚠️ 一律使用物理坐标**：传给 ljqCtrl.Click/SetCursorPos 的坐标必须是物理坐标（=截图像素坐标）。禁止传入逻辑坐标。
|
||||
- **物理验证**：模拟操作前必须确保窗口已通过 `activate()` 置于前台。
|
||||
- **偏移量**：所有的相对偏移像素值（如“向右移动 10 像素”）同样需要除以 `dpi_scale`。
|
||||
- **坐标对齐**: 物理坐标 = 截图坐标；ljqCtrl 自动处理 DPI 换算，禁止手动重复计算。
|
||||
- **⚠️ 窗口坐标转换陷阱**：使用 `win32gui.GetWindowRect(hwnd)` 获取的矩形包含标题栏和边框，而截图内容是客户区。点击截图内元素时，必须用 `win32gui.ClientToScreen(hwnd, (0, 0))` 获取客户区原点的屏幕坐标，再加上截图内坐标。禁止直接用 GetWindowRect 左上角 + 截图坐标。
|
||||
- **⚠️ win32 DPI 坐标陷阱**：未调用 `SetProcessDPIAware()` 时，`GetWindowRect/ClientToScreen/GetClientRect` 等拿到的窗口/客户区坐标通常是**逻辑坐标**；若后续截图或 `ljqCtrl` 使用的是物理像素，必须统一做 `坐标 / ljqCtrl.dpi_scale`。等价方案：先 `SetProcessDPIAware()`，之后全流程直接使用 raw 物理坐标，禁止逻辑/物理坐标混用。
|
||||
- **⚠️ win32 DPI 坐标陷阱**：未调用 `SetProcessDPIAware()` 时，`GetWindowRect/ClientToScreen/GetClientRect` 等拿到的窗口/客户区坐标通常是**逻辑坐标**，必须进行换算！
|
||||
- **文本输入**：ljqCtrl 无 TypeText/SendKeys。向输入框键入文本：先点击/三击选中字段，再 `pyperclip.copy('文本'); ljqCtrl.Press('ctrl+v')`。
|
||||
+3
-1
@@ -15,6 +15,8 @@ from ultralytics import YOLO
|
||||
from PIL import Image, ImageDraw
|
||||
import numpy as np
|
||||
|
||||
print('[UI DETECT] 截图分析后必须使用物理坐标，ljqCtrl也使用物理坐标！')
|
||||
|
||||
DEFAULT_MODEL = str(Path(__file__).resolve().parent.parent / 'temp' / 'weights' / 'icon_detect' / 'model.pt')
|
||||
|
||||
try:
|
||||
@@ -128,7 +130,7 @@ def detect(image_path, mode='match', model_path=None, conf=0.25, iou_thresh=0.5)
|
||||
for i, (ox1, oy1, ox2, oy2, text, oc) in enumerate(ocr_items):
|
||||
if i not in matched_ocr:
|
||||
elements.append({'bbox': [ox1,oy1,ox2,oy2], 'type': 'text', 'label': text, 'confidence': oc})
|
||||
|
||||
#if [x for x in elements if x['label'] is None]: print('[TIPS] crop grid + VLM to identify target no text icon if needed')
|
||||
return elements
|
||||
|
||||
def visualize(image_path, elements, output_path=None):
|
||||
|
||||
Reference in new issue
Block a user