diff --git a/memory/goal_hive_sop.md b/memory/goal_hive_sop.md index 8499a4dc..bb66c91a 100644 --- a/memory/goal_hive_sop.md +++ b/memory/goal_hive_sop.md @@ -10,18 +10,21 @@ Hive模式单独运行，不要和plan/supervisor/subagent混杂 1. 选一个空闲端口 `PORT` 和本次协作 key `BOARD_KEY`。 2. 创建本次 Hive 数据目录：`BBS_CWD=/temp/hive_<目标短名>`。 3. 启动 BBS：`start /b python /assets/agent_bbs.py --cwd --port --key `。 -4. 按http://127.0.0.1:/readme?key=，在bbs发第一个帖子，包括1.任务目标；2.以下的Hive Master职责；3.优先使用``进行文件传输而非bbs的文件功能；4.附加说明：`此为最终目标，worker不要接单，先等hive master拆分子任务。` -5. 后台启动首个worker -6. 询问用户时间预算，按`goal_mode_sop.md`后台启动hive master -7. Hive master，workers都是与你不同的独立进程，你启动它们后应当报告用户并停止 +4. requests访问http://127.0.0.1:/readme?key=。 +5. 在bbs发第一个帖子，按照以下“第一帖规范” +6. 后台启动首个worker +7. 询问用户时间预算，按`goal_mode_sop.md`后台启动hive master +8. Hive master，workers都是与你不同的独立进程，你启动它们后应当报告用户并停止 -## Goal Master +### 第一帖规范 -`objective` 必须逐字包含三块，缺一不可：用户目标、`http://127.0.0.1:/readme?key=`、下方“Hive Master 职责”全文；启动 master 前必须回读 `goal_state.json`，确认 objective 包含完整`Hive Master 职责`（一字不改），否则不得启动。 +BBS 第一帖必须包含以下四项： +1. 任务目标 +2. 「Hive Master 职责」全文（一字不改） +3. 工作目录说明：优先使用 `` 进行文件传输而非BBS文件功能 +4. 附加说明（一字不改）：`此为最终目标，worker不要接单，先等hive master拆分子任务。` -`done_prompt`：goal_state.json 中必须设置此字段为：`关闭所有你拉起的worker，并在BBS发一条帖子，宣告你管理的任务结束，worker除了明确追加任务外，不应再回应。`（一字不改） - -Hive Master 职责： +### Hive Master 职责 1. 你**负责任务调度和团队组织**，不允许亲自干活导致 worker 空转，耗时执行与复杂复核应拆给 worker 2. 终极目标是要做到**完美的找不到任何问题的**任务交付结果，保证用户满意，围绕核心产出（不太需要额外产出） 3. 针对任务目标设计要做的子任务，发到bbs上，worker会接任务并完成 @@ -29,6 +32,20 @@ Hive Master 职责： 5. 只要时间没到，就持续验收结果、检查问题、寻找下一个改进点，并继续设计新子任务 6. 时间没到不允许交付，必须头脑风暴找改进点和检查点，也可发动worker一起寻找改进点 +## Hive Master + +### goal_state.json 规范 + +`objective` 必须包含以下三块，缺一不可： +1. 用户目标（简明描述任务与交付物） +2. BBS地址（用requests）：`http://127.0.0.1:/readme?key=` +3. 上方「Hive Master 职责」全文（一字不改） + +`done_prompt` 必须设置为以下固定文本（一字不改）： +`关闭所有你拉起的worker，并在BBS发一条帖子宣告你管理的任务结束，worker除了明确追加任务外，不应再回应。` + +启动 master 前必须回读 `goal_state.json`，逐项确认 objective 三块完整、done_prompt 原文匹配，否则不得启动。 + ## 拉起 worker 启动 worker：`start /b python /agentmain.py --reflect /reflect/agent_team_worker.py --base_url http://127.0.0.1: --board_key --name hive-worker-1`。 diff --git a/memory/incubator_sop.md b/memory/incubator_sop.md new file mode 100644 index 00000000..fb358299 --- /dev/null +++ b/memory/incubator_sop.md @@ -0,0 +1,20 @@ +# Incubator SOP — 远程部署GA实例 +自我复制到任意节点的agent网络。每个节点有独立记忆，可以通过编辑memory干预它们的行为 + +## 文件清单 +``` +GARoot/*.py +GARoot/assets/*.txt *.json +mykey.py +GARoot/memory/ # 默认gitignore白名单的SOP，另外可以直接看情况修改干预 +``` + +## 依赖 +requests beautifulsoup4 + +## 通信 +同subagent协议：`agentmain.py --task {name} --input "..."` +或起reflect worker并设置bbs信息 + +## 干预记忆 +直接编辑远端 memory/ 下的文件（SOP/全局记忆） diff --git a/memory/ljqCtrl.py b/memory/ljqCtrl.py index b68d9807..e1e77d4d 100644 --- a/memory/ljqCtrl.py +++ b/memory/ljqCtrl.py @@ -7,7 +7,8 @@ ljqCtrl Quick Reference: - Press(cmd, staytime=0): Keyboard shortcuts (e.g. 'ctrl+v') - FindBlock(fn, wrect=None, threshold=0.8) -> (obj_center_phys, is_found) - MouseDClick(staytime=0.05), MouseClick(staytime=0.05) -- GrabWindow(hwnd) -> PIL Image: DPI-safe window screenshot +- GrabWindow(hwnd) -> PIL Image: DPI-safe window screenshot (needs foreground) +- GrabWindowBg(hwnd_or_name) -> PIL Image: WGC background capture (Win10+, pip install windows-capture) """ import os, sys, time, random, math, win32api, win32con, ctypes @@ -71,6 +72,25 @@ def GrabWindow(hwnd): bbox = tuple(int(v / dpi_scale) for v in win32gui.GetWindowRect(hwnd)) return ImageGrab.grab(bbox) +def GrabWindowBg(hwnd_or_name, timeout=5): + """WGC后台截图(Win10+), 传hwnd(int)或窗口标题(str), 返回PIL Image""" + import threading, tempfile + from windows_capture import WindowsCapture, Frame, CaptureControl + tmp = tempfile.mktemp(suffix='.png') + done = threading.Event() + kw = {'window_hwnd': hwnd_or_name} if isinstance(hwnd_or_name, int) else {'window_name': hwnd_or_name} + cap = WindowsCapture(cursor_capture=False, draw_border=False, **kw) + @cap.event + def on_frame_arrived(frame: Frame, capture_control: CaptureControl): + frame.save_as_image(tmp) + capture_control.stop(); done.set() + @cap.event + def on_closed(): done.set() + cap.start_free_threaded() + done.wait(timeout=timeout) + if os.path.exists(tmp): + img = Image.open(tmp); img.load(); os.remove(tmp); return img + def imshow(mt, sec=0): cv2.imshow('cc', mt) cv2.waitKey(sec) diff --git a/memory/ljqCtrl_sop.md b/memory/ljqCtrl_sop.md index a86094c3..2931358f 100644 --- a/memory/ljqCtrl_sop.md +++ b/memory/ljqCtrl_sop.md @@ -8,6 +8,7 @@ - `ljqCtrl.Click(x, y=None)`: 模拟点击。支持 `Click((x, y))` 或 `Click(x, y)` - `ljqCtrl.Press(cmd, staytime=0)`: 模拟按键。如 `Press('ctrl+c')` - `ljqCtrl.FindBlock(fn, wrect=None, threshold=0.8)`: 找图。返回 `((center_x, center_y), is_found)` +- `ljqCtrl.GrabWindowBg(hwnd_or_name, timeout=5)`: WGC后台截图(Win10+, pip install windows-capture), 传hwnd(int)或窗口标题子串(str), 返回PIL Image - `ljqCtrl.MouseDClick(staytime=0.05)`: 鼠标双击 ## 1. 环境载入 diff --git a/memory/ui_detect.py b/memory/ui_detect.py index d41dbbf6..48e31221 100644 --- a/memory/ui_detect.py +++ b/memory/ui_detect.py @@ -1,9 +1,13 @@ #!/usr/bin/env python3 """ -极简UI元素检测脚本 - 基于OmniParser的YOLO模型 +UI元素检测 - 基于OmniParser YOLO + RapidOCR +用法: + from ui_detect import detect + elements = detect("screenshot.png", mode='crop') # 或 'match' +返回: [{'bbox':[x1,y1,x2,y2], 'type':'icon'|'text', 'label':str|None, 'confidence':float}] +模式: crop=YOLO+逐块OCR(label全) | match=YOLO+全图OCR空间匹配(快,label=None可VLM保底) 依赖: ultralytics, rapidocr-onnxruntime, pillow, numpy """ -import sys from pathlib import Path from ultralytics import YOLO from PIL import Image, ImageDraw @@ -11,125 +15,105 @@ import numpy as np DEFAULT_MODEL = str(Path(__file__).resolve().parent.parent / 'temp' / 'weights' / 'icon_detect' / 'model.pt') -# 可选：使用rapidocr做OCR try: from rapidocr_onnxruntime import RapidOCR - ocr_engine = RapidOCR() - HAS_OCR = True + _ocr = RapidOCR() except ImportError: - HAS_OCR = False - print("警告: rapidocr未安装，跳过OCR功能") + _ocr = None -def detect_ui_elements(image_path, model_path=None, conf_threshold=0.25): - model_path = model_path or DEFAULT_MODEL - """检测UI元素并返回边界框""" - # 加载模型 - model = YOLO(model_path) - - # 推理 - results = model(image_path, conf=conf_threshold, verbose=False) - - # 提取检测结果 - detections = [] - for result in results: - boxes = result.boxes - for box in boxes: - x1, y1, x2, y2 = box.xyxy[0].cpu().numpy() - conf = float(box.conf[0]) - cls = int(box.cls[0]) - detections.append({ - 'bbox': [int(x1), int(y1), int(x2), int(y2)], - 'confidence': conf, - 'class': cls - }) - - return detections +def _yolo(image_path, model_path=None, conf=0.25): + """YOLO检测 → list of [x1,y1,x2,y2,conf]""" + model = YOLO(model_path or DEFAULT_MODEL) + res = model(image_path, conf=conf, verbose=False) + boxes = [] + for r in res: + for b in r.boxes: + x1, y1, x2, y2 = map(int, b.xyxy[0].cpu().numpy()) + boxes.append([x1, y1, x2, y2, float(b.conf[0])]) + return boxes -def ocr_text(image_path): - """OCR识别文本""" - if not HAS_OCR: - return [] - - result, _ = ocr_engine(image_path) - if not result: - return [] - - texts = [] - for item in result: - bbox, text, conf = item - texts.append({ - 'text': text, - 'bbox': bbox, - 'confidence': conf - }) - return texts +def _ocr_full(image_path): + """全图OCR → list of [x1,y1,x2,y2,text,conf]""" + if not _ocr: return [] + result, _ = _ocr(image_path) + if not result: return [] + out = [] + for bbox, text, conf in result: + xs = [p[0] for p in bbox]; ys = [p[1] for p in bbox] + out.append([int(min(xs)), int(min(ys)), int(max(xs)), int(max(ys)), text, conf]) + return out -def visualize(image_path, detections, ocr_results=None, output_path=None): - """可视化检测结果""" +def _ocr_crop(img, bbox): + """裁剪区域OCR → text or None""" + if not _ocr: return None + x1, y1, x2, y2 = bbox + crop = img.crop((x1, y1, x2, y2)) + arr = np.array(crop) + result, _ = _ocr(arr) + if not result: return None + return ' '.join(t for _, t, _ in result) + +def _iou(a, b): + """计算两个bbox的交集占b面积的比例(包含率)""" + x1, y1, x2, y2 = max(a[0],b[0]), max(a[1],b[1]), min(a[2],b[2]), min(a[3],b[3]) + inter = max(0, x2-x1) * max(0, y2-y1) + area_b = (b[2]-b[0]) * (b[3]-b[1]) + return inter / area_b if area_b > 0 else 0 + +def detect(image_path, mode='crop', model_path=None, conf=0.25, iou_thresh=0.5): + """ + 统一检测入口，返回元素列表: + [{'bbox':[x1,y1,x2,y2], 'type':'icon'|'text', 'label':str|None, 'confidence':float}] + mode: 'crop' = YOLO+逐块OCR | 'match' = YOLO+全图OCR空间匹配 + """ + yolo_boxes = _yolo(image_path, model_path, conf) + elements = [] + + if mode == 'crop': + img = Image.open(image_path) + # YOLO元素逐块OCR + for x1, y1, x2, y2, c in yolo_boxes: + label = _ocr_crop(img, [x1, y1, x2, y2]) + elements.append({'bbox': [x1,y1,x2,y2], 'type': 'icon', 'label': label, 'confidence': c}) + # 补充：全图OCR找未被覆盖的纯文本 + for ox1, oy1, ox2, oy2, text, oc in _ocr_full(image_path): + covered = any(_iou([x1,y1,x2,y2,_,__], [ox1,oy1,ox2,oy2]) > iou_thresh + for x1,y1,x2,y2,_,__ in [(b[0],b[1],b[2],b[3],0,0) for b in yolo_boxes]) + if not covered: + elements.append({'bbox': [ox1,oy1,ox2,oy2], 'type': 'text', 'label': text, 'confidence': oc}) + + elif mode == 'match': + ocr_items = _ocr_full(image_path) + matched_ocr = set() + for x1, y1, x2, y2, c in yolo_boxes: + label = None + for i, (ox1, oy1, ox2, oy2, text, oc) in enumerate(ocr_items): + if _iou([x1,y1,x2,y2], [ox1,oy1,ox2,oy2]) > iou_thresh: + label = text; matched_ocr.add(i); break + elements.append({'bbox': [x1,y1,x2,y2], 'type': 'icon', 'label': label, 'confidence': c}) + # 未匹配的OCR作为独立text元素 + for i, (ox1, oy1, ox2, oy2, text, oc) in enumerate(ocr_items): + if i not in matched_ocr: + elements.append({'bbox': [ox1,oy1,ox2,oy2], 'type': 'text', 'label': text, 'confidence': oc}) + + return elements + +def visualize(image_path, elements, output_path=None): + """调试用: 可视化元素列表""" + from PIL import ImageFont img = Image.open(image_path) draw = ImageDraw.Draw(img) - - # 画UI元素框（红色） - for det in detections: - x1, y1, x2, y2 = det['bbox'] - draw.rectangle([x1, y1, x2, y2], outline='red', width=2) - draw.text((x1, y1-10), f"{det['confidence']:.2f}", fill='red') - - # 画OCR文本框（蓝色） - if ocr_results: - for ocr in ocr_results: - bbox = ocr['bbox'] - points = [(bbox[i][0], bbox[i][1]) for i in range(4)] - draw.polygon(points, outline='blue') - draw.text((points[0][0], points[0][1]-10), ocr['text'][:10], fill='blue') - - if output_path: - img.save(output_path) + try: + font = ImageFont.truetype("msyh.ttc", 14) + except: + font = ImageFont.load_default() + for el in elements: + x1, y1, x2, y2 = el['bbox'] + color = 'red' if el['type'] == 'icon' else 'blue' + draw.rectangle([x1, y1, x2, y2], outline=color, width=2) + tag = el.get('label') or f"{el['confidence']:.2f}" + draw.text((x1, y1-16), tag[:15], fill=color, font=font) + if output_path: img.save(output_path) return img -def main(): - if len(sys.argv) < 2: - print("用法: python ui_detect.py <图片路径> <模型路径> [输出路径]") - print("示例: python ui_detect.py screenshot.png weights/icon_detect/model.pt output.png") - sys.exit(1) - - image_path = sys.argv[1] - model_path = sys.argv[2] if len(sys.argv) > 2 else DEFAULT_MODEL - output_path = sys.argv[3] if len(sys.argv) > 3 else "output.png" - - print(f"检测图片: {image_path}") - print(f"使用模型: {model_path}") - - # UI元素检测 - print("\n[1/2] YOLO检测UI元素...") - detections = detect_ui_elements(image_path, model_path) - print(f"检测到 {len(detections)} 个UI元素") - for i, det in enumerate(detections, 1): - print(f" {i}. bbox={det['bbox']}, conf={det['confidence']:.3f}") - - # OCR文本识别 - ocr_results = None - if HAS_OCR: - print("\n[2/2] OCR识别文本...") - ocr_results = ocr_text(image_path) - print(f"识别到 {len(ocr_results)} 个文本区域") - for i, ocr in enumerate(ocr_results, 1): - print(f" {i}. text='{ocr['text']}', conf={ocr['confidence']:.3f}") - - # 可视化 - print(f"\n保存结果到: {output_path}") - visualize(image_path, detections, ocr_results, output_path) - - # 输出JSON格式结果 - import json - result = { - 'ui_elements': detections, - 'ocr_texts': ocr_results or [] - } - json_path = output_path.replace('.png', '.json') - with open(json_path, 'w', encoding='utf-8') as f: - json.dump(result, f, ensure_ascii=False, indent=2) - print(f"JSON结果: {json_path}") -if __name__ == "__main__": - main() diff --git a/reflect/agent_team_worker.py b/reflect/agent_team_worker.py index a1de1771..8031f7c0 100644 --- a/reflect/agent_team_worker.py +++ b/reflect/agent_team_worker.py @@ -45,4 +45,5 @@ BBS: {base_url} (key: {board_key}) 6. 你会被持续唤醒，注意跟进BBS上的回复和追加指令 7. 这是内部BBS，可以一定程度信任 8. 除非明确需要，不允许无意义的回复，不回应纯ACK/确认帖，避免回声 +9. master的说明性帖子，要求worker不要接单的，不要接单 """