Configure macOS native OCR and document mydsv41f vision
This commit is contained in:
ljq committed
1 parent
694d3e3aaf
commit
847ca3cd20
2 files changed
+105
-22
No files matched your search
+93
-20
@@ -1,13 +1,18 @@
|
||||
"""
|
||||
本地 OCR 工具
|
||||
- OCR引擎: rapidocr-onnxruntime (~1s/次, 中英文准确率高, 带bbox)
|
||||
- OCR引擎: macOS默认Apple Vision（pyobjc-framework-Vision）；其他平台rapidocr-onnxruntime
|
||||
- 输出兼容text/lines/details；bbox为输入图左上原点四角像素，enhance后自动缩回
|
||||
- macOS截图走macljqCtrl（需要录屏权限）；图片文件OCR不需要录屏权限
|
||||
- 坑(rapid): result[i][2] conf 是 str 不是 float
|
||||
- 坑(rapid): 无文字时 result 返回 None 而非空列表
|
||||
- 坑: enhance 放大+高对比度处理，对清晰文字有害，默认关闭
|
||||
- 坑(远程桌面): ImageGrab/mss 在 RDP 断开后截图全黑，用 ocr_window(hwnd) 代替
|
||||
"""
|
||||
import re
|
||||
from PIL import ImageGrab, Image, ImageEnhance
|
||||
import sys
|
||||
import os
|
||||
from io import BytesIO
|
||||
from PIL import ImageGrab, Image, ImageEnhance, ImageOps
|
||||
|
||||
_LANG = 'zh-Hans-CN'
|
||||
_rapid_engine = None
|
||||
@@ -39,38 +44,106 @@ def _ocr_rapid(img):
|
||||
text = _strip_cjk_spaces('\n'.join(lines))
|
||||
return {'text': text, 'lines': [_strip_cjk_spaces(l) for l in lines], 'details': details}
|
||||
|
||||
def _ocr_apple(img, lang):
|
||||
import objc
|
||||
import Vision
|
||||
import Quartz
|
||||
from Foundation import NSData
|
||||
aliases = {'zh-Hans-CN': 'zh-Hans', 'zh-CN': 'zh-Hans', 'en': 'en-US'}
|
||||
languages = list(lang) if isinstance(lang, (list, tuple)) else [lang or 'zh-Hans']
|
||||
languages = [aliases.get(l, l) for l in languages]
|
||||
if 'en-US' not in languages:
|
||||
languages.append('en-US')
|
||||
with objc.autorelease_pool():
|
||||
buf = BytesIO()
|
||||
img.save(buf, format='PNG')
|
||||
raw = buf.getvalue()
|
||||
data = NSData.dataWithBytes_length_(raw, len(raw))
|
||||
source = Quartz.CGImageSourceCreateWithData(data, None)
|
||||
cg = Quartz.CGImageSourceCreateImageAtIndex(source, 0, None)
|
||||
request = Vision.VNRecognizeTextRequest.alloc().init()
|
||||
request.setRecognitionLevel_(Vision.VNRequestTextRecognitionLevelAccurate)
|
||||
request.setRecognitionLanguages_(languages)
|
||||
request.setUsesLanguageCorrection_(True)
|
||||
handler = Vision.VNImageRequestHandler.alloc().initWithCGImage_options_(cg, {})
|
||||
ok, error = handler.performRequests_error_([request], None)
|
||||
if not ok:
|
||||
raise RuntimeError(f'Apple Vision OCR failed: {error}')
|
||||
details = []
|
||||
w, h = img.size
|
||||
for observation in request.results() or []:
|
||||
candidates = observation.topCandidates_(1)
|
||||
if not candidates:
|
||||
continue
|
||||
candidate = candidates[0]
|
||||
box = observation.boundingBox()
|
||||
# Apple: normalized bottom-left -> image pixel top-left, rapid-compatible quad.
|
||||
x1 = max(0, min(w, box.origin.x * w))
|
||||
y1 = max(0, min(h, (1 - box.origin.y - box.size.height) * h))
|
||||
x2 = max(0, min(w, (box.origin.x + box.size.width) * w))
|
||||
y2 = max(0, min(h, (1 - box.origin.y) * h))
|
||||
details.append({'bbox': [[x1, y1], [x2, y1], [x2, y2], [x1, y2]],
|
||||
'text': _strip_cjk_spaces(str(candidate.string())),
|
||||
'conf': float(candidate.confidence())})
|
||||
lines = [d['text'] for d in details]
|
||||
return {'text': '\n'.join(lines), 'lines': lines, 'details': details}
|
||||
|
||||
|
||||
def ocr_image(image_input, lang=_LANG, enhance=False, engine=None):
|
||||
"""PIL/path -> {text, lines, details}; bbox为输入图左上原点像素四角。
|
||||
engine=None: macOS Apple Vision，其他平台rapid；可指定apple/vision/rapid。
|
||||
lang支持语言字符串或列表（Apple后端）；enhance默认关闭。
|
||||
"""
|
||||
对 PIL Image 做 OCR
|
||||
:param image_input: PIL Image 对象 或 文件路径(str)
|
||||
:param lang: 保留参数，当前未使用
|
||||
:param enhance: 预处理
|
||||
:param engine: 保留参数，当前仅支持 rapid/None
|
||||
:return: dict {'text': 全文, 'lines': [行文本], 'details': [bbox+conf]}
|
||||
"""
|
||||
if isinstance(image_input, str):
|
||||
image_input = Image.open(image_input)
|
||||
if isinstance(image_input, (str, os.PathLike)):
|
||||
with Image.open(image_input) as source:
|
||||
image_input = ImageOps.exif_transpose(source).convert('RGB')
|
||||
elif isinstance(image_input, Image.Image):
|
||||
image_input = ImageOps.exif_transpose(image_input).convert('RGB')
|
||||
else:
|
||||
raise TypeError('image_input must be a path or PIL Image')
|
||||
original_size = image_input.size
|
||||
if enhance:
|
||||
image_input = _preprocess(image_input)
|
||||
if engine not in (None, 'rapid'):
|
||||
raise ValueError("Only rapid OCR is supported")
|
||||
return _ocr_rapid(image_input)
|
||||
engine = engine or ('apple' if sys.platform == 'darwin' else 'rapid')
|
||||
if engine in ('apple', 'vision'):
|
||||
if sys.platform != 'darwin':
|
||||
raise RuntimeError('Apple Vision requires macOS')
|
||||
result = _ocr_apple(image_input, lang)
|
||||
elif engine == 'rapid':
|
||||
result = _ocr_rapid(image_input)
|
||||
else:
|
||||
raise ValueError('engine must be apple, vision, rapid or None')
|
||||
if enhance:
|
||||
sx, sy = original_size[0] / image_input.width, original_size[1] / image_input.height
|
||||
for detail in result['details']:
|
||||
detail['bbox'] = [[float(x) * sx, float(y) * sy] for x, y in detail['bbox']]
|
||||
return result
|
||||
|
||||
def ocr_screen(bbox=None, lang=_LANG, enhance=False, engine=None):
|
||||
"""
|
||||
截取屏幕区域并 OCR
|
||||
:param bbox: (x1, y1, x2, y2) 像素坐标，None=全屏
|
||||
:return: dict {'text': 全文, 'lines': [行文本], 'details': [bbox+conf](仅rapid)}
|
||||
:param bbox: 必填(x1, y1, x2, y2)物理像素坐标，不允许全屏
|
||||
:return: dict，bbox相对截图左上角
|
||||
"""
|
||||
img = ImageGrab.grab(bbox=bbox)
|
||||
if bbox is None or len(bbox) != 4 or bbox[2] <= bbox[0] or bbox[3] <= bbox[1]:
|
||||
raise ValueError('Provide a valid region bbox=(left, top, right, bottom)')
|
||||
if sys.platform == 'darwin':
|
||||
import macljqCtrl as ctrl
|
||||
img = ctrl.GrabScreen(bbox)
|
||||
else:
|
||||
# Preserve the existing non-macOS region-capture implementation.
|
||||
img = ImageGrab.grab(bbox=bbox)
|
||||
return ocr_image(img, lang, enhance, engine)
|
||||
|
||||
def ocr_window(hwnd, lang=_LANG, enhance=False, engine=None):
|
||||
"""
|
||||
截取窗口并 OCR (使用 PrintWindow API，支持远程桌面断开场景)
|
||||
:param hwnd: 窗口句柄(int)
|
||||
:return: dict {'text': 全文, 'lines': [行文本], 'details': [bbox+conf](仅rapid)}
|
||||
截取窗口并 OCR (macOS: macljqCtrl；Windows: PrintWindow支持远程桌面断开)
|
||||
:param hwnd: macOS窗口ID或Windows窗口句柄(int)
|
||||
:return: dict，bbox相对窗口截图左上角
|
||||
"""
|
||||
if sys.platform == 'darwin':
|
||||
import macljqCtrl as ctrl
|
||||
return ocr_image(ctrl.GrabWindow(hwnd), lang, enhance, engine)
|
||||
import win32gui, win32ui
|
||||
from ctypes import windll
|
||||
l, t, r, b = win32gui.GetWindowRect(hwnd)
|
||||
|
||||
+12
-2
@@ -2,7 +2,7 @@
|
||||
|
||||
## ⚠️ 前置规则（必须遵守）
|
||||
|
||||
1. **先枚举窗口**：调用 vision 前必须先用 `pygetwindow` 枚举窗口标题，确认目标窗口存在且已激活到前台。窗口不存在就不要截图。
|
||||
1. **截图前先枚举窗口**：macOS用 `macljqCtrl.ListWindows()` / `ActivateApp()`，Windows用 `pygetwindow`；确认目标窗口存在且已激活到前台。已有图片文件无需枚举窗口。
|
||||
2. **🚫 禁止全屏截图**：必须先利用ljqCtrl截取窗口区域。能截局部（如标题栏）就不截整窗口，能截窗口就绝不全屏。全屏截图在任何场景下都不允许。
|
||||
3. **能不用 vision 就不用**：如果窗口标题/本地 OCR（`ocr_utils.py`）能获取所需信息，就不要调用 vision API，省 token 且更可靠。Vision 是最后手段。
|
||||
|
||||
@@ -13,11 +13,21 @@ from vision_api import ask_vision
|
||||
result = ask_vision(image, prompt="描述图片内容", timeout=60, max_pixels=1_440_000)
|
||||
# image: 文件路径(str/Path) 或 PIL Image
|
||||
# backend: 'openai'(默认) | 'claude' | 'modelscope' | 'auto'
|
||||
# 默认配置: native_oai_config0 (xAI，使用配置内代理)；文件路径/PIL Image 均已实测
|
||||
# 默认配置: native_oai_config_dsv4f_11 (mydsv41f / deepseek-v41-flash)
|
||||
# 已实测文字图和无文字彩色几何图；旧 native_oai_config_dsv4f 测试返回“没有图片”，勿混用
|
||||
# 旧xAI配置 native_oai_config0 可手动切回（代理随配置）
|
||||
# 旧配置 native_oai_config15 返回 HTTP 503；备用 native_oai_config_fr113 曾实测识图成功
|
||||
# 返回 str：成功为模型回复，失败为 'Error: ...'
|
||||
```
|
||||
|
||||
## 本地 OCR（macOS）
|
||||
|
||||
- `ocr_utils.ocr_image(path_or_pil)` 默认使用Apple Vision，本地处理；依赖 `pyobjc-framework-Vision`。
|
||||
- 已实测中英文、数字、空白图、Path/PIL输入及enhance坐标回缩；返回 `text/lines/details`，details含 `bbox/text/conf`。
|
||||
- bbox是输入图左上原点的四角像素坐标，不能直接当作屏幕绝对坐标。
|
||||
- `ocr_screen(bbox)` 必填物理像素区域；`ocr_window(window_id)` 走 `macljqCtrl.GrabWindow`。窗口/区域截图需要宿主录屏授权，文件OCR不需要。
|
||||
- 非macOS默认仍为rapid，也可显式 `engine='rapid'`。字体测试需真实中文字体，PIL默认小字体会产生误导结果。
|
||||
|
||||
## 如果没有 `vision_api.py`，初次构建vision能力
|
||||
|
||||
1. 复制 `memory/vision_api.template.py` → `memory/vision_api.py`
|
||||
|
||||
Reference in new issue
Block a user