From 0b8ade048edd4b8456fc5f8ea5f6e9c9470446c4 Mon Sep 17 00:00:00 2001 From: Chengfang Lu Date: Tue, 16 Jun 2026 09:58:35 +0800 Subject: [PATCH] feat: Unified Ingest SDK for DMS/ADAS promote, cuboid export and 3D fit Replace subprocess build with promote_batch SDK, add ADAS cuboid export/fit/validate pipeline, stage normalization, and offline unit tests wired into smoke_labeling_api. Co-authored-by: Cursor --- algorithms/adas_mono3d/__init__.py | 0 algorithms/adas_mono3d/fit_cuboid.py | 146 ++++++++++++ as.py | 94 +++++--- datasets/dms/scripts/ingest_incremental.py | 35 +++ datasets/labeling.registry.yaml | 2 +- docs/ADAS_MOON3D_PACK.md | 60 +++++ docs/LABELING_SOP.md | 8 +- manifests/repo_layout.json | 2 +- platform/as_platform/api/labeling_routes.py | 26 +++ platform/as_platform/api/server.py | 14 +- platform/as_platform/api/system_routes.py | 41 ++++ platform/as_platform/audit/preview.py | 29 +++ platform/as_platform/audit/queue.py | 3 +- platform/as_platform/audit/review.py | 5 +- platform/as_platform/data/promote/__init__.py | 11 + .../as_platform/data/promote/adas_cuboid.py | 152 +++++++++++++ platform/as_platform/data/promote/base.py | 56 +++++ platform/as_platform/data/promote/dms_yolo.py | 62 +++++ platform/as_platform/data/promote/manifest.py | 93 ++++++++ platform/as_platform/data/promote/registry.py | 18 ++ platform/as_platform/data/promote/runner.py | 126 +++++++++++ .../data/promote/validate/__init__.py | 0 .../data/promote/validate/adas_cuboid.py | 81 +++++++ .../data/promote/validate/dms_yolo.py | 18 ++ platform/as_platform/jobs/queue.py | 4 + platform/as_platform/jobs/runner.py | 86 +++++-- platform/as_platform/labeling/batch_stage.py | 92 +++++++- platform/as_platform/labeling/class_map.py | 74 ++++++ .../labeling/export_cuboid_batch.py | 174 ++++++++++++++ .../as_platform/labeling/fit_cuboid_batch.py | 95 ++++++++ .../as_platform/labeling/format_converter.py | 65 ++++++ platform/as_platform/labeling/service.py | 50 +++- platform/as_platform/labeling/stage.py | 29 +++ .../tests/test_unified_ingest_sdk.py | 213 ++++++++++++++++++ platform/web/src/app/hsap-api.ts | 6 + .../modules/labeling/pages/CampaignsPage.tsx | 10 +- .../src/modules/labeling/pages/ExportPage.tsx | 107 +++++++-- .../labeling/pages/QualityReviewPage.tsx | 8 +- .../modules/labeling/pages/WorkbenchPage.tsx | 6 + scripts/smoke_adas_promote.sh | 63 ++++++ scripts/smoke_labeling_api.sh | 6 + workflow.registry.yaml | 8 +- 42 files changed, 2074 insertions(+), 104 deletions(-) create mode 100644 algorithms/adas_mono3d/__init__.py create mode 100644 algorithms/adas_mono3d/fit_cuboid.py create mode 100644 docs/ADAS_MOON3D_PACK.md create mode 100644 platform/as_platform/data/promote/__init__.py create mode 100644 platform/as_platform/data/promote/adas_cuboid.py create mode 100644 platform/as_platform/data/promote/base.py create mode 100644 platform/as_platform/data/promote/dms_yolo.py create mode 100644 platform/as_platform/data/promote/manifest.py create mode 100644 platform/as_platform/data/promote/registry.py create mode 100644 platform/as_platform/data/promote/runner.py create mode 100644 platform/as_platform/data/promote/validate/__init__.py create mode 100644 platform/as_platform/data/promote/validate/adas_cuboid.py create mode 100644 platform/as_platform/data/promote/validate/dms_yolo.py create mode 100644 platform/as_platform/labeling/class_map.py create mode 100644 platform/as_platform/labeling/export_cuboid_batch.py create mode 100644 platform/as_platform/labeling/fit_cuboid_batch.py create mode 100644 platform/as_platform/labeling/stage.py create mode 100644 platform/as_platform/tests/test_unified_ingest_sdk.py create mode 100755 scripts/smoke_adas_promote.sh diff --git a/algorithms/adas_mono3d/__init__.py b/algorithms/adas_mono3d/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/algorithms/adas_mono3d/fit_cuboid.py b/algorithms/adas_mono3d/fit_cuboid.py new file mode 100644 index 0000000..120283c --- /dev/null +++ b/algorithms/adas_mono3d/fit_cuboid.py @@ -0,0 +1,146 @@ +"""Cuboid 16pt + K → MOON-3D 3D detection fields (MVP fit).""" +from __future__ import annotations + +import math +from typing import Any + +# Default WLH priors in meters (width, length, height) — BK2/MOON convention +CLASS_PRIORS: dict[str, tuple[float, float, float]] = { + "pedestrian": (0.6, 0.6, 1.7), + "car": (1.8, 4.5, 1.5), + "truck": (2.5, 8.0, 3.0), + "bus": (2.5, 10.0, 3.2), + "motorcycle": (0.8, 2.0, 1.5), + "tricycle": (1.2, 2.5, 1.8), + "traffic cone": (0.4, 0.4, 0.8), +} + + +def cuboid_points_to_box2d(points: list[float]) -> list[float] | None: + if len(points) < 16: + return None + xs = [float(points[i]) for i in range(0, 16, 2)] + ys = [float(points[i]) for i in range(1, 16, 2)] + return [min(xs), min(ys), max(xs), max(ys)] + + +def _project_point(x: float, y: float, z: float, K: list[list[float]]) -> tuple[float, float]: + fx, fy = float(K[0][0]), float(K[1][1]) + cx, cy = float(K[0][2]), float(K[1][2]) + if z <= 0.01: + z = 0.01 + u = fx * x / z + cx + v = fy * y / z + cy + return u, v + + +def _reproj_error(center, wlh, K, points2d) -> float: + cx, cy, cz = center + w, l, h = wlh + # 8 corners in object frame (simplified axis-aligned box, no rotation MVP) + corners = [ + (-l / 2, -w / 2, -h / 2), (l / 2, -w / 2, -h / 2), + (l / 2, w / 2, -h / 2), (-l / 2, w / 2, -h / 2), + (-l / 2, -w / 2, h / 2), (l / 2, -w / 2, h / 2), + (l / 2, w / 2, h / 2), (-l / 2, w / 2, h / 2), + ] + # camera: x right, y down, z forward — object x forward, y left, z up + err = 0.0 + for i, (ox, oy, oz) in enumerate(corners): + cam_x = -oy + cam_y = -oz + cam_z = ox + cz + u, v = _project_point(cam_x, cam_y, cam_z, K) + px = points2d[i * 2] + py = points2d[i * 2 + 1] + err += (u - px) ** 2 + (v - py) ** 2 + return err / max(len(corners), 1) + + +def fit_cuboid_detection( + points: list[float], + K: list[list[float]], + class_name: str, +) -> dict[str, Any]: + """Fit 3D box from 16 cuboid points. Returns fields to merge into detection.""" + box2d = cuboid_points_to_box2d(points) + if not box2d or not K: + return {"fit_ok": False, "fit_error": "missing points or K"} + + w0, l0, h0 = CLASS_PRIORS.get(class_name.lower(), CLASS_PRIORS.get(class_name, (1.8, 4.0, 1.5))) + if class_name.lower() not in {k.lower() for k in CLASS_PRIORS}: + for k, v in CLASS_PRIORS.items(): + if k.lower() == class_name.lower(): + w0, l0, h0 = v + break + + fx = float(K[0][0]) + fy = float(K[1][1]) + cy = float(K[1][2]) + y1, y2 = box2d[1], box2d[3] + pix_h = max(y2 - y1, 1.0) + # depth from pinhole: h_pix = fy * H / Z + z_est = fy * h0 / pix_h + x1, x2 = box2d[0], box2d[2] + u_c = (x1 + x2) / 2.0 + cx_cam = (u_c - float(K[0][2])) * z_est / fx + + # grid search depth / center for min reprojection error + best_err = float("inf") + best = (cx_cam, 0.0, z_est, w0, l0, h0) + for dz in (-0.3, -0.15, 0, 0.15, 0.3): + for dy in (-0.5, 0, 0.5): + z = max(z_est + dz * z_est, 1.0) + cx = cx_cam + dy + err = _reproj_error((cx, 0.0, z), (w0, l0, h0), K, points) + if err < best_err: + best_err = err + best = (cx, 0.0, z, w0, l0, h0) + + cx, cy, cz, w, l, h = best + # OpenCV camera: x right, y down, z forward + center_3d = [float(cx), float(cy), float(cz)] + dimensions_wlh = [float(w), float(l), float(h)] + rot_y = 0.0 + qw = math.cos(rot_y / 2) + qy = math.sin(rot_y / 2) + quaternion_wxyz = [float(qw), 0.0, float(qy), 0.0] + + fit_ok = best_err < 50000.0 # pixel^2 threshold MVP + return { + "center_3d": center_3d, + "dimensions_wlh": dimensions_wlh, + "quaternion_wxyz": quaternion_wxyz, + "rotation_y": rot_y, + "fit_ok": fit_ok, + "fit_error": float(best_err), + "box2d_xyxy": box2d, + } + + +def fit_quaternion_json_file(data: dict[str, Any]) -> dict[str, Any]: + K = data.get("K") + if not K: + return data + out_dets = [] + for det in data.get("detections") or []: + det = dict(det) + if det.get("fit_ok"): + out_dets.append(det) + continue + # recover points from box2d if no cuboid points stored — skip 3D + class_name = str(det.get("class_name") or "car") + box = det.get("box2d_xyxy") + if not box or len(box) < 4: + out_dets.append(det) + continue + # synthetic 16pt from AABB (degenerate but allows fit attempt) + x1, y1, x2, y2 = box[:4] + pts = [x1, y1, x2, y1, x1, y2, x2, y2, x1, y1, x2, y1, x1, y2, x2, y2] + fitted = fit_cuboid_detection(pts, K, class_name) + det.update({k: v for k, v in fitted.items() if k != "box2d_xyxy"}) + out_dets.append(det) + data = dict(data) + data["detections"] = out_dets + data["num_detections"] = len(out_dets) + return data diff --git a/as.py b/as.py index 06c5091..b528e23 100755 --- a/as.py +++ b/as.py @@ -272,18 +272,14 @@ def cmd_build_dms( all_sources: bool, skip_validate: bool, ) -> None: - root = proj_root(wf, "dms") - scripts = root / "scripts" - if not task: cmd_refresh_dms(wf, None) print("已按 active_packs 生成 manifests/yaml_active/*.yaml（未合并任何新文件）") return - ensure_dms_pack(root, pack) - scripts = root / "scripts" - if all_sources: + root = proj_root(wf, "dms") + scripts = root / "scripts" cmd = [ sys.executable, str(scripts / "ingest_incremental.py"), @@ -291,42 +287,59 @@ def cmd_build_dms( "--pack", pack, "--all-sources", ] - elif batch: - src = root / "inbox" / task / batch - if not src.is_dir(): - sys.exit(f"inbox 批次不存在: {src}") - cmd = [ - sys.executable, - str(scripts / "ingest_incremental.py"), - "--task", task, - "--pack", pack, - "--src", str(src), - ] - else: - cmd = [ - sys.executable, - str(scripts / "ingest_incremental.py"), - "--task", task, - "--pack", pack, - "--all-inbox", - ] - if dry_run: - cmd.append("--dry-run") - subprocess.check_call(cmd, cwd=root) - - if dry_run: + if dry_run: + cmd.append("--dry-run") + subprocess.check_call(cmd, cwd=root) + if not dry_run and not skip_validate: + run_validate_dms(task) + if refresh and not dry_run: + cmd_refresh_dms(wf, task) return - if not skip_validate: - print("validate …") - run_validate_dms(task) + from as_platform.data.promote.runner import promote_batch - if refresh: + result = promote_batch( + "dms", + task=task, + batch=batch, + pack=pack, + dry_run=dry_run, + skip_validate=skip_validate, + refresh=refresh, + ) + print(json.dumps(result, ensure_ascii=False, indent=2)) + if not dry_run and not skip_validate: + run_validate_dms(task) + if refresh and not dry_run: cmd_refresh_dms(wf, task) - else: + elif not dry_run: print("提示: python as.py build dms --refresh # 生成训练 yaml") +def cmd_build_adas( + wf: dict, + task: str, + batch: str | None, + pack: str, + dry_run: bool, + skip_validate: bool, +) -> None: + if not batch: + sys.exit("adas build 需要 --batch") + from as_platform.data.promote.runner import promote_batch + + result = promote_batch( + "adas", + task=task, + batch=batch, + pack=pack, + dry_run=dry_run, + skip_validate=skip_validate, + allow_partial_3d=True, + ) + print(json.dumps(result, ensure_ascii=False, indent=2)) + + def cmd_eval_dms(wf: dict, task: str, weights: Path | None, save_candidate: bool) -> None: cmd = [sys.executable, str(WORKSPACE / "scripts" / "eval_dms.py"), task] if weights: @@ -550,7 +563,7 @@ def main() -> None: ad.add_argument("--copy", action="store_true") bd = sub.add_parser("build") - bd.add_argument("project", choices=("dms", "lane")) + bd.add_argument("project", choices=("dms", "lane", "adas")) bd.add_argument("task", nargs="?") bd.add_argument("--pack", default="dms_v1", help="dms 写入/合并的目标包") bd.add_argument("--batch") @@ -634,6 +647,15 @@ def main() -> None: args.all_sources, getattr(args, "skip_validate", False), ) + elif args.project == "adas": + cmd_build_adas( + wf, + args.task or "cuboid_7cls", + args.batch, + args.pack or "adas_moon3d_v1", + args.dry_run, + getattr(args, "skip_validate", False), + ) else: cmd_build_lane(wf) elif args.cmd == "eval": diff --git a/datasets/dms/scripts/ingest_incremental.py b/datasets/dms/scripts/ingest_incremental.py index 4a9103e..169b51c 100644 --- a/datasets/dms/scripts/ingest_incremental.py +++ b/datasets/dms/scripts/ingest_incremental.py @@ -557,5 +557,40 @@ def main() -> None: print("提示: 可运行 python scripts/refresh_yaml.py") +def promote_inbox_batch( + *, + root: Path, + task: str, + pack: str, + src: Path, + mode: str | None = None, + dry_run: bool = False, + refresh: bool = True, + copy: bool = False, +) -> dict: + """Programmatic inbox batch promote (used by Pack Promote SDK).""" + reg = load_registry(root.resolve()) + split_cfg = reg.get("split") or {} + ns = argparse.Namespace( + task=task, + submode=mode, + mode=mode, + pack=pack, + dry_run=dry_run, + copy=copy, + to="train", + val_ratio=float(split_cfg.get("val_ratio", 0.1)), + seed=int(split_cfg.get("seed", 42)), + resplit=bool(split_cfg.get("resplit_after_ingest", True)), + dedup="stem", + ) + result = ingest_one(root.resolve(), reg, task, src.resolve(), ns) + if not dry_run: + append_log(root.resolve(), {"src": str(src), "pack": pack, **result}) + if refresh: + run_refresh(root.resolve()) + return result + + if __name__ == "__main__": main() diff --git a/datasets/labeling.registry.yaml b/datasets/labeling.registry.yaml index 3859d6e..ef52f93 100644 --- a/datasets/labeling.registry.yaml +++ b/datasets/labeling.registry.yaml @@ -38,8 +38,8 @@ profiles: export_default: cvat_cuboid ml_adapter: adas_yolo26 cvat_labels: - - car - pedestrian + - car - truck - bus - motorcycle diff --git a/docs/ADAS_MOON3D_PACK.md b/docs/ADAS_MOON3D_PACK.md new file mode 100644 index 0000000..accb4dc --- /dev/null +++ b/docs/ADAS_MOON3D_PACK.md @@ -0,0 +1,60 @@ +# ADAS MOON-3D 训练包 `adas_moon3d_v1` + +## 目录结构 + +```text +datasets/adas/packs/adas_moon3d_v1/ +├── sources/{batch}/ +│ ├── images/ +│ ├── calib/ +│ └── labels/quaternion_json/ +├── lists/ +│ ├── train_stems.txt +│ └── val_stems.txt +└── manifests/ + └── pack_index.yaml +``` + +## class_id（BK2/MOON 顺序） + +| ID | 类别 | +|----|------| +| 0 | pedestrian | +| 1 | car | +| 2 | truck | +| 3 | bus | +| 4 | motorcycle | +| 5 | tricycle | +| 6 | traffic cone | + +定义于 [`data/送标/adas/adas.registry.yaml`](../../data/送标/adas/adas.registry.yaml) 与 [`datasets/labeling.registry.yaml`](../datasets/labeling.registry.yaml)。 + +## 管线 + +1. **labeling_export** — CVAT ls_annotations → `labels/quaternion_json/*.json` +2. **cuboid_fit_3d**（有 calib 时自动触发）— 补全 3D 字段 +3. **build_adas**（审核）— `promote_batch` 复制到 pack + 刷新 stem 列表 + +## CLI + +```bash +# 导出（平台 Job 或脚本） +PYTHONPATH=platform python3 -c "from as_platform.labeling.export_cuboid_batch import export_batch; ..." + +# 3D 拟合 +PYTHONPATH=platform python3 -c "from as_platform.labeling.fit_cuboid_batch import fit_batch; ..." + +# 入包 +python as.py build adas cuboid_7cls --batch val_front6mm_pilot --pack adas_moon3d_v1 +``` + +## Smoke + +```bash +bash scripts/smoke_adas_promote.sh +``` + +## 与 dms/packs/adas_v1 的区别 + +- `dms/packs/adas_v1`：2D YOLO 历史包（[`scripts/organize_adas.py`](../scripts/organize_adas.py)） +- `datasets/adas/packs/adas_moon3d_v1`：MOON-3D quaternion_json 3D GT diff --git a/docs/LABELING_SOP.md b/docs/LABELING_SOP.md index 407833d..86babaa 100644 --- a/docs/LABELING_SOP.md +++ b/docs/LABELING_SOP.md @@ -10,8 +10,12 @@ |------|------|-----------------| | `raw_pool` | 待标注原图 | 有 `images/`，无 `labels/` | | `out_for_labeling` | 送标中 | 已 open Campaign | -| `returned` | 回传待入库 | 有 `images/` + `labels/`（YOLO txt） | -| `ingested` | 已入库 | 已进入 pack/sources 或 Lane gt | +| `returned` | 回传待 build | 有导出产物（DMS: YOLO txt；ADAS: quaternion_json） | +| `ingested` | 已入库 | 已进入 pack/sources（`promote_batch`） | + +**协调员 QA 门禁路径（推荐）：** 标完 → **提交质检** (`in_review`) → 质检通过 (`labeling_submitted`) → **执行导出** → `returned` → **提交 build**（审核）→ `ingested`。 + +ADAS cuboid 详见 [ADAS_MOON3D_PACK.md](./ADAS_MOON3D_PACK.md)：导出 → 可选 `cuboid_fit_3d` → `build_adas` 进 `adas_moon3d_v1`。 **登记 meta**（`更多操作` 内）只写入 `batch.meta.yaml`，**不等于**已送标。 diff --git a/manifests/repo_layout.json b/manifests/repo_layout.json index 7acfab6..41ecf1b 100644 --- a/manifests/repo_layout.json +++ b/manifests/repo_layout.json @@ -1,5 +1,5 @@ { "layout": "workspace_symlinks", "workspace": "/data/workspace", - "linked_at": "2026-06-15T07:37:38+00:00" + "linked_at": "2026-06-16T01:56:18+00:00" } diff --git a/platform/as_platform/api/labeling_routes.py b/platform/as_platform/api/labeling_routes.py index 108695d..51a263d 100644 --- a/platform/as_platform/api/labeling_routes.py +++ b/platform/as_platform/api/labeling_routes.py @@ -37,6 +37,8 @@ from as_platform.labeling.service import ( open_campaign, submit_campaign, trigger_labeling_export, + get_batch_export_stats, + trigger_cuboid_fit, ) from as_platform.labeling.vendor_import import import_vendor_zip, list_registry_profiles @@ -315,6 +317,30 @@ def api_labeling_export( raise HTTPException(404, "campaign not found") from None +@router.get("/api/v1/labeling/campaigns/{campaign_id}/export-stats") +def api_batch_export_stats( + campaign_id: str, + _user: Annotated[User, Depends(require_permission("read:pending"))], +) -> dict[str, Any]: + try: + return get_batch_export_stats(campaign_id) + except FileNotFoundError: + raise HTTPException(404, "campaign not found") from None + + +@router.post("/api/v1/labeling/campaigns/{campaign_id}/cuboid-fit") +def api_cuboid_fit( + campaign_id: str, + _user: Annotated[User, Depends(require_permission("read:pending"))], +) -> dict[str, Any]: + try: + return trigger_cuboid_fit(campaign_id) + except FileNotFoundError: + raise HTTPException(404, "campaign not found") from None + except ValueError as e: + raise HTTPException(400, str(e)) from e + + @router.get("/api/v1/labeling/campaigns/{campaign_id}/export-jobs") def api_campaign_export_jobs( campaign_id: str, diff --git a/platform/as_platform/api/server.py b/platform/as_platform/api/server.py index 2969491..713fd9e 100644 --- a/platform/as_platform/api/server.py +++ b/platform/as_platform/api/server.py @@ -498,15 +498,23 @@ def api_submit(body: SubmitApprovalBody, user: Annotated[User, Depends(get_curre @app.post("/api/v1/approvals/submit-build-batch") def api_submit_build_batch(body: BuildFromBatchBody, user: Annotated[User, Depends(get_current_user)]) -> dict[str, Any]: - if not can_submit_action(user, "build_dms"): + action = "build_adas" if body.project == "adas" else "build_dms" + if not can_submit_action(user, action) and not can_submit_action(user, "build_dms"): raise HTTPException(403, "无权提交 build") - params: dict[str, Any] = {"task": body.task, "pack": body.pack} + pack = body.pack + if body.project == "adas" and (not pack or pack == "dms_v2"): + pack = "adas_moon3d_v1" + params: dict[str, Any] = { + "project": body.project, + "task": body.task, + "pack": pack, + } if body.location == "inbox": params["batch"] = body.batch else: params["all_sources"] = True return submit_approval( - "build_dms", params, + action, params, submitted_by=user.name, submitted_by_user_id=user.id, note=body.note or f"入库 {body.batch}", diff --git a/platform/as_platform/api/system_routes.py b/platform/as_platform/api/system_routes.py index 91ce6a2..daeacdc 100644 --- a/platform/as_platform/api/system_routes.py +++ b/platform/as_platform/api/system_routes.py @@ -92,6 +92,47 @@ def api_system_submit_approval( raise HTTPException(400, str(e)) from e +class BuildFromBatchBody(BaseModel): + project: str = "dms" + task: str + batch: str + pack: str = "dms_v2" + location: str = "inbox" + note: str | None = None + + +@router.post("/audit/submit-build-batch") +def api_system_submit_build_batch( + body: BuildFromBatchBody, + user: Annotated[User, Depends(get_current_user)], +) -> dict[str, Any]: + action = "build_adas" if body.project == "adas" else "build_dms" + if not can_submit_action(user, action) and not can_submit_action(user, "build_dms"): + raise HTTPException(403, "无权提交 build") + pack = body.pack + if body.project == "adas" and (not pack or pack == "dms_v2"): + pack = "adas_moon3d_v1" + params: dict[str, Any] = { + "project": body.project, + "task": body.task, + "pack": pack, + } + if body.location == "inbox": + params["batch"] = body.batch + else: + params["all_sources"] = True + try: + return submit_approval( + action, + params, + submitted_by=user.name, + submitted_by_user_id=user.id, + note=body.note or f"入库 {body.batch}", + ) + except ValueError as e: + raise HTTPException(400, str(e)) from e + + @router.get("/audit/{record_id}") def api_system_get_approval( record_id: str, diff --git a/platform/as_platform/audit/preview.py b/platform/as_platform/audit/preview.py index 58af559..9ba3ff7 100644 --- a/platform/as_platform/audit/preview.py +++ b/platform/as_platform/audit/preview.py @@ -264,6 +264,35 @@ def resolve_approval_scope(action: str, params: dict[str, Any]) -> dict[str, Any "batches": batches, } + if action == "build_adas": + task = p.get("task") or "cuboid_7cls" + batch_name = p.get("batch") + root = proj_root(wf, "adas") + batches: list[dict[str, Any]] = [] + if batch_name: + batches.append({"path": root / "inbox" / task / batch_name, "batch": batch_name, "location": "inbox"}) + pack = p.get("pack") or "adas_moon3d_v1" + stats: dict[str, Any] = {} + if batch_name: + from as_platform.data.promote.validate.adas_cuboid import validate_adas_cuboid_batch + + bpath = root / "inbox" / task / batch_name + if bpath.is_dir(): + _err, _warn, stats = validate_adas_cuboid_batch(bpath, allow_partial_3d=True) + from as_platform.labeling.class_map import load_adas_class_names + + names = load_adas_class_names() + class_names = {i: n for i, n in enumerate(names)} + return { + "project": "adas", + "task": task, + "pack": pack, + "scope_label": f"ADAS · {task} · {pack}" + (f" · {batch_name}" if batch_name else ""), + "class_names": class_names, + "batches": batches, + "export_stats": stats, + } + if action == "delivery_ingest": data_path = (p.get("data_path") or "").strip() if not data_path: diff --git a/platform/as_platform/audit/queue.py b/platform/as_platform/audit/queue.py index 54a1095..fe534ac 100644 --- a/platform/as_platform/audit/queue.py +++ b/platform/as_platform/audit/queue.py @@ -11,7 +11,7 @@ from as_platform.config import LANE_DATA_VIZ_ENABLED from as_platform.integrations.feishu_notify import send_chat_async ACTIONS_REQUIRING_APPROVAL = { - "build_dms", "build_lane", "enable_pack", "disable_pack", + "build_dms", "build_adas", "build_lane", "enable_pack", "disable_pack", "train_dms", "train_lane", "eval_dms", "promote_dms", "pipeline_dms", "register_batch", "eval_lane", "visualize_dms", "visualize_lane", "delivery_ingest", @@ -31,6 +31,7 @@ REJECTION_CATEGORY_LABEL = {k: v for k, v in REJECTION_CATEGORIES.items()} ACTION_LABELS = { "build_dms": "DMS 入库 (build)", + "build_adas": "ADAS 入库 (build)", "build_lane": "车道线合并列表 (build lane)", "enable_pack": "启用训练数据包", "disable_pack": "停用训练数据包", diff --git a/platform/as_platform/audit/review.py b/platform/as_platform/audit/review.py index f92fb33..3a8eb65 100644 --- a/platform/as_platform/audit/review.py +++ b/platform/as_platform/audit/review.py @@ -272,9 +272,10 @@ def _update_campaign_stage(db, campaign_id: str, new_stage: str) -> None: from as_platform.labeling.batch_stage import update_campaign_batch_meta_stage camp = db.get(LabelingCampaign, campaign_id) if camp: - camp.status = new_stage + effective = "labeling_submitted" if new_stage == "review_approved" else new_stage + camp.status = effective db.flush() - update_campaign_batch_meta_stage(camp, new_stage) + update_campaign_batch_meta_stage(camp, effective) def review_progress(campaign_id: str) -> dict[str, int]: diff --git a/platform/as_platform/data/promote/__init__.py b/platform/as_platform/data/promote/__init__.py new file mode 100644 index 0000000..d3c03ae --- /dev/null +++ b/platform/as_platform/data/promote/__init__.py @@ -0,0 +1,11 @@ +from as_platform.data.promote.base import PackPromoteAdapter, PromoteContext, PromoteResult +from as_platform.data.promote.registry import get_promote_adapter +from as_platform.data.promote.runner import promote_batch + +__all__ = [ + "PackPromoteAdapter", + "PromoteContext", + "PromoteResult", + "get_promote_adapter", + "promote_batch", +] diff --git a/platform/as_platform/data/promote/adas_cuboid.py b/platform/as_platform/data/promote/adas_cuboid.py new file mode 100644 index 0000000..30dc7b5 --- /dev/null +++ b/platform/as_platform/data/promote/adas_cuboid.py @@ -0,0 +1,152 @@ +"""ADAS cuboid MOON-3D pack promote adapter.""" +from __future__ import annotations + +import json +import os +import shutil +from datetime import datetime, timezone +from pathlib import Path + +from as_platform.data.batch import read_meta, write_meta +from as_platform.data.promote.base import PackPromoteAdapter, PromoteContext, PromoteResult +from as_platform.data.promote.manifest import refresh_adas_lists +from as_platform.data.promote.validate.adas_cuboid import validate_adas_cuboid_batch +from as_platform.labeling.class_map import build_class_map, load_adas_class_names, normalize_detection_class + +IMG_EXTS = {".jpg", ".jpeg", ".png", ".bmp", ".webp"} + + +def _link_or_copy(src: Path, dst: Path, *, copy: bool = False) -> None: + dst.parent.mkdir(parents=True, exist_ok=True) + if dst.exists() or dst.is_symlink(): + dst.unlink() + if copy: + if src.is_dir(): + shutil.copytree(src, dst, dirs_exist_ok=True) + else: + shutil.copy2(src, dst) + return + try: + os.link(src, dst) + except OSError: + if src.is_dir(): + shutil.copytree(src, dst, dirs_exist_ok=True) + else: + shutil.copy2(src, dst) + + +def _sync_tree(src: Path, dst: Path, *, copy: bool = False) -> int: + count = 0 + if not src.is_dir(): + return 0 + for p in sorted(src.rglob("*")): + if not p.is_file(): + continue + rel = p.relative_to(src) + target = dst / rel + if not target.exists(): + _link_or_copy(p, target, copy=copy) + count += 1 + return count + + +def _normalize_quaternion_json(dest_batch: Path) -> int: + qdir = dest_batch / "labels" / "quaternion_json" + if not qdir.is_dir(): + return 0 + cmap = build_class_map(load_adas_class_names()) + names = load_adas_class_names() + updated = 0 + for p in qdir.glob("*.json"): + data = json.loads(p.read_text(encoding="utf-8")) + dets = [] + for det in data.get("detections") or []: + dets.append(normalize_detection_class(det, cmap)) + data["detections"] = dets + data["text_prompts"] = names + data["num_detections"] = len(dets) + p.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8") + updated += 1 + return updated + + +class AdasCuboidPromoteAdapter(PackPromoteAdapter): + project = "adas" + + def validate(self, ctx: PromoteContext) -> list[str]: + if ctx.skip_validate: + return [] + errors, warnings, _stats = validate_adas_cuboid_batch( + ctx.batch_dir, + allow_partial_3d=ctx.allow_partial_3d, + ) + ctx.extra.setdefault("validate_warnings", warnings) + return errors + + def promote(self, ctx: PromoteContext) -> PromoteResult: + warnings = list(ctx.extra.get("validate_warnings") or []) + qdir = ctx.batch_dir / "labels" / "quaternion_json" + if not qdir.is_dir() or not any(qdir.glob("*.json")): + return PromoteResult( + ok=False, + project=ctx.project, + task=ctx.task, + batch=ctx.batch, + pack=ctx.pack, + warnings=["missing quaternion_json export"], + ) + + pack_dir = ctx.project_root / "packs" / ctx.pack + dest = pack_dir / "sources" / ctx.batch + if ctx.dry_run: + return PromoteResult( + ok=True, + project=ctx.project, + task=ctx.task, + batch=ctx.batch, + pack=ctx.pack, + dest_path=str(dest), + detail={"dry_run": True}, + ) + + if dest.exists(): + shutil.rmtree(dest) + dest.mkdir(parents=True, exist_ok=True) + + copied = 0 + for sub in ("images", "calib", "labels"): + src_sub = ctx.batch_dir / sub + if src_sub.is_dir(): + copied += _sync_tree(src_sub, dest / sub) + + normalized = _normalize_quaternion_json(dest) + + meta = read_meta(ctx.batch_dir) or {} + meta.update({ + "stage": "ingested", + "project": ctx.project, + "task": ctx.task, + "batch": ctx.batch, + "pack": ctx.pack, + "ingested_at": datetime.now(timezone.utc).isoformat(), + "pipeline_version": 2, + }) + write_meta(dest, meta) + write_meta(ctx.batch_dir, meta) + + manifest = refresh_adas_lists(pack=ctx.pack) + img_count = sum(1 for _ in (dest / "images").rglob("*") if _.suffix.lower() in IMG_EXTS) if (dest / "images").is_dir() else 0 + + return PromoteResult( + ok=True, + project=ctx.project, + task=ctx.task, + batch=ctx.batch, + pack=ctx.pack, + dest_path=str(dest), + images=img_count, + labels=normalized, + manifest_paths=[manifest.get("train_list", ""), manifest.get("val_list", "")], + warnings=warnings, + detail={"copied_files": copied, "normalized_json": normalized, **manifest}, + ) diff --git a/platform/as_platform/data/promote/base.py b/platform/as_platform/data/promote/base.py new file mode 100644 index 0000000..1e284f7 --- /dev/null +++ b/platform/as_platform/data/promote/base.py @@ -0,0 +1,56 @@ +"""Pack promote adapter base types.""" +from __future__ import annotations + +from abc import ABC, abstractmethod +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Any + + +@dataclass +class PromoteContext: + project: str + task: str + batch: str + pack: str + batch_dir: Path + project_root: Path + dry_run: bool = False + skip_validate: bool = False + allow_partial_3d: bool = False + refresh: bool = True + extra: dict[str, Any] = field(default_factory=dict) + + +@dataclass +class PromoteResult: + ok: bool + project: str + task: str + batch: str + pack: str + dest_path: str = "" + images: int = 0 + labels: int = 0 + manifest_paths: list[str] = field(default_factory=list) + warnings: list[str] = field(default_factory=list) + stage: str = "ingested" + detail: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + out = asdict(self) + out["ok"] = self.ok + return out + + +class PackPromoteAdapter(ABC): + project: str = "" + + @abstractmethod + def promote(self, ctx: PromoteContext) -> PromoteResult: + raise NotImplementedError + + @abstractmethod + def validate(self, ctx: PromoteContext) -> list[str]: + """Return list of error messages; empty means pass.""" + raise NotImplementedError diff --git a/platform/as_platform/data/promote/dms_yolo.py b/platform/as_platform/data/promote/dms_yolo.py new file mode 100644 index 0000000..cffb341 --- /dev/null +++ b/platform/as_platform/data/promote/dms_yolo.py @@ -0,0 +1,62 @@ +"""DMS YOLO pack promote adapter.""" +from __future__ import annotations + +import sys +from pathlib import Path + +from as_platform.data.promote.base import PackPromoteAdapter, PromoteContext, PromoteResult +from as_platform.data.promote.manifest import refresh_dms_yaml +from as_platform.data.promote.validate.dms_yolo import validate_dms_task + +_DMS_SCRIPTS = Path(__file__).resolve().parents[4] / "datasets" / "dms" / "scripts" +if str(_DMS_SCRIPTS) not in sys.path: + sys.path.insert(0, str(_DMS_SCRIPTS)) + + +class DmsYoloPromoteAdapter(PackPromoteAdapter): + project = "dms" + + def validate(self, ctx: PromoteContext) -> list[str]: + if ctx.skip_validate: + return [] + return validate_dms_task(ctx.task) + + def promote(self, ctx: PromoteContext) -> PromoteResult: + from ingest_incremental import promote_inbox_batch + + if not ctx.batch_dir.is_dir(): + return PromoteResult( + ok=False, + project=ctx.project, + task=ctx.task, + batch=ctx.batch, + pack=ctx.pack, + warnings=[f"batch_dir missing: {ctx.batch_dir}"], + ) + + pack_dir = ctx.project_root / "packs" / ctx.pack + pack_dir.mkdir(parents=True, exist_ok=True) + + detail = promote_inbox_batch( + root=ctx.project_root, + task=ctx.task, + pack=ctx.pack, + src=ctx.batch_dir, + mode=ctx.extra.get("mode"), + dry_run=ctx.dry_run, + refresh=ctx.refresh and not ctx.dry_run, + ) + if ctx.refresh and not ctx.dry_run and not ctx.skip_validate: + refresh_dms_yaml(task=ctx.task) + + added = int(detail.get("added") or 0) + return PromoteResult( + ok=True, + project=ctx.project, + task=ctx.task, + batch=ctx.batch, + pack=ctx.pack, + dest_path=str(ctx.project_root / "packs" / ctx.pack), + labels=added, + detail=detail, + ) diff --git a/platform/as_platform/data/promote/manifest.py b/platform/as_platform/data/promote/manifest.py new file mode 100644 index 0000000..50bd884 --- /dev/null +++ b/platform/as_platform/data/promote/manifest.py @@ -0,0 +1,93 @@ +"""Refresh ADAS / DMS pack manifests after promote.""" +from __future__ import annotations + +import json +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import yaml + +from as_platform.data.core import load_wf, proj_root + + +def _collect_adas_stems(sources_root: Path) -> list[str]: + stems: list[str] = [] + if not sources_root.is_dir(): + return stems + for batch_dir in sorted(sources_root.iterdir()): + if not batch_dir.is_dir() or batch_dir.name.startswith("."): + continue + qdir = batch_dir / "labels" / "quaternion_json" + if qdir.is_dir(): + for p in sorted(qdir.glob("*.json")): + stems.append(p.stem) + else: + img_root = batch_dir / "images" + if img_root.is_dir(): + for p in sorted(img_root.rglob("*")): + if p.is_file() and p.suffix.lower() in {".jpg", ".jpeg", ".png"}: + stems.append(p.stem) + return sorted(set(stems)) + + +def refresh_adas_lists(wf: dict | None = None, *, pack: str = "adas_moon3d_v1") -> dict[str, Any]: + wf = wf or load_wf() + root = proj_root(wf, "adas") + pack_dir = root / "packs" / pack + sources = pack_dir / "sources" + lists_dir = pack_dir / "lists" + lists_dir.mkdir(parents=True, exist_ok=True) + + stems = _collect_adas_stems(sources) + val_ratio = 0.1 + reg_path = root / wf["projects"]["adas"]["registry"] + if reg_path.is_file(): + reg = yaml.safe_load(reg_path.read_text(encoding="utf-8")) or {} + val_ratio = float((reg.get("split") or {}).get("val_ratio", 0.1)) + + n_val = max(0, int(len(stems) * val_ratio)) if len(stems) > 1 else 0 + val_stems = stems[:n_val] + train_stems = stems[n_val:] + + train_path = lists_dir / "train_stems.txt" + val_path = lists_dir / "val_stems.txt" + train_path.write_text("\n".join(train_stems) + ("\n" if train_stems else ""), encoding="utf-8") + val_path.write_text("\n".join(val_stems) + ("\n" if val_stems else ""), encoding="utf-8") + + manifest_dir = pack_dir / "manifests" + manifest_dir.mkdir(parents=True, exist_ok=True) + index_path = manifest_dir / "pack_index.yaml" + batches = [] + if sources.is_dir(): + for d in sorted(sources.iterdir()): + if d.is_dir() and not d.name.startswith("."): + batches.append({"batch": d.name, "path": str(d)}) + index = { + "pack": pack, + "updated_at": datetime.now(timezone.utc).isoformat(), + "batches": batches, + "train_stems": len(train_stems), + "val_stems": len(val_stems), + } + index_path.write_text(yaml.dump(index, allow_unicode=True, sort_keys=False), encoding="utf-8") + + return { + "train_list": str(train_path), + "val_list": str(val_path), + "pack_index": str(index_path), + "train_count": len(train_stems), + "val_count": len(val_stems), + } + + +def refresh_dms_yaml(wf: dict | None = None, task: str | None = None) -> None: + wf = wf or load_wf() + root = proj_root(wf, "dms") + import subprocess + import sys + + cmd = [sys.executable, str(root / "scripts" / "refresh_yaml.py")] + if task: + cmd.extend(["--task", task]) + subprocess.check_call(cmd, cwd=str(root)) diff --git a/platform/as_platform/data/promote/registry.py b/platform/as_platform/data/promote/registry.py new file mode 100644 index 0000000..aecbd14 --- /dev/null +++ b/platform/as_platform/data/promote/registry.py @@ -0,0 +1,18 @@ +"""Pack promote adapter registry.""" +from __future__ import annotations + +from as_platform.data.promote.adas_cuboid import AdasCuboidPromoteAdapter +from as_platform.data.promote.base import PackPromoteAdapter +from as_platform.data.promote.dms_yolo import DmsYoloPromoteAdapter + +ADAPTERS: tuple[PackPromoteAdapter, ...] = ( + DmsYoloPromoteAdapter(), + AdasCuboidPromoteAdapter(), +) + + +def get_promote_adapter(project: str) -> PackPromoteAdapter: + for adapter in ADAPTERS: + if adapter.project == project: + return adapter + raise ValueError(f"no promote adapter for project={project}") diff --git a/platform/as_platform/data/promote/runner.py b/platform/as_platform/data/promote/runner.py new file mode 100644 index 0000000..4fa5eaa --- /dev/null +++ b/platform/as_platform/data/promote/runner.py @@ -0,0 +1,126 @@ +"""Unified pack promote entrypoint.""" +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from as_platform.data.batch import read_meta, write_meta +from as_platform.data.catalog_cache import invalidate_catalog_cache +from as_platform.data.core import load_wf, proj_root +from as_platform.data.promote.base import PromoteContext, PromoteResult +from as_platform.data.promote.registry import get_promote_adapter +from as_platform.db.engine import session_scope +from as_platform.db.models import LabelingCampaign +from as_platform.jobs.runner import _auto_snapshot +from as_platform.labeling.annotate import resolve_campaign_batch_dir + + +def _resolve_batch_dir( + project: str, + task: str, + batch: str, + *, + location: str = "inbox", +) -> Path: + wf = load_wf() + root = proj_root(wf, project) + if location == "inbox": + if project == "adas": + return (root / "inbox" / task / batch).resolve() + return (root / "inbox" / task / batch).resolve() + raise ValueError(f"unsupported location: {location}") + + +def _update_campaign_ingested(project: str, task: str, batch: str) -> None: + try: + with session_scope() as db: + camp = ( + db.query(LabelingCampaign) + .filter( + LabelingCampaign.project == project, + LabelingCampaign.task == task, + LabelingCampaign.batch == batch, + ) + .order_by(LabelingCampaign.created_at.desc()) + .first() + ) + if camp: + camp.status = "ingested" + db.flush() + try: + batch_dir = resolve_campaign_batch_dir(camp) + meta = read_meta(batch_dir) or {} + meta["stage"] = "ingested" + meta["pipeline_version"] = 2 + write_meta(batch_dir, meta) + except Exception: + pass + except Exception: + pass + + +def promote_batch( + project: str, + *, + task: str, + batch: str | None = None, + pack: str | None = None, + batch_dir: Path | str | None = None, + dry_run: bool = False, + skip_validate: bool = False, + allow_partial_3d: bool = False, + refresh: bool = True, + all_sources: bool = False, + extra: dict[str, Any] | None = None, +) -> dict[str, Any]: + """Promote inbox batch into training pack (SDK entry).""" + wf = load_wf() + pcfg = wf["projects"][project] + pack_name = pack or pcfg.get("base_pack") + if not pack_name: + raise ValueError(f"project {project} missing pack") + if not task: + raise ValueError("task required") + if all_sources: + raise ValueError("all_sources promote not yet in SDK; use CLI ingest_incremental") + if not batch: + raise ValueError("batch required") + + root = proj_root(wf, project) + bdir = Path(batch_dir).resolve() if batch_dir else _resolve_batch_dir(project, task, batch) + if not bdir.is_dir(): + raise ValueError(f"batch_dir not found: {bdir}") + + adapter = get_promote_adapter(project) + ctx = PromoteContext( + project=project, + task=task, + batch=batch, + pack=pack_name, + batch_dir=bdir, + project_root=root, + dry_run=dry_run, + skip_validate=skip_validate, + allow_partial_3d=allow_partial_3d, + refresh=refresh, + extra=extra or {}, + ) + + val_errors = adapter.validate(ctx) + if val_errors: + raise ValueError("; ".join(val_errors)) + + result: PromoteResult = adapter.promote(ctx) + if not result.ok: + raise ValueError(result.warnings[0] if result.warnings else "promote failed") + + if not dry_run: + _update_campaign_ingested(project, task, batch) + invalidate_catalog_cache() + if project == "dms": + _auto_snapshot("dms", task=task) + + out = result.to_dict() + out["stdout"] = __import__("json").dumps(out, ensure_ascii=False) + out["stderr"] = "" + return out diff --git a/platform/as_platform/data/promote/validate/__init__.py b/platform/as_platform/data/promote/validate/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/platform/as_platform/data/promote/validate/adas_cuboid.py b/platform/as_platform/data/promote/validate/adas_cuboid.py new file mode 100644 index 0000000..b5e1fba --- /dev/null +++ b/platform/as_platform/data/promote/validate/adas_cuboid.py @@ -0,0 +1,81 @@ +"""ADAS cuboid batch validation before promote.""" +from __future__ import annotations + +import json +from pathlib import Path + +from as_platform.labeling.class_map import load_adas_class_names + + +def validate_adas_cuboid_batch( + batch_dir: Path, + *, + allow_partial_3d: bool = False, + min_fit_ratio: float = 0.8, +) -> tuple[list[str], list[str], dict]: + """Return (errors, warnings, stats).""" + errors: list[str] = [] + warnings: list[str] = [] + qdir = batch_dir / "labels" / "quaternion_json" + expected_names = load_adas_class_names() + + if not qdir.is_dir(): + errors.append(f"missing labels/quaternion_json under {batch_dir}") + return errors, warnings, {} + + files = sorted(qdir.glob("*.json")) + if not files: + errors.append("no quaternion_json files") + return errors, warnings, {} + + total_dets = 0 + fit_ok = 0 + has_k = 0 + files_with_dets = 0 + for p in files: + try: + data = json.loads(p.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as e: + errors.append(f"{p.name}: invalid json ({e})") + continue + dets = data.get("detections") or [] + if not dets: + warnings.append(f"{p.name}: empty detections (skipped)") + continue + files_with_dets += 1 + if data.get("K"): + has_k += 1 + prompts = data.get("text_prompts") or [] + if prompts and list(prompts) != expected_names: + warnings.append(f"{p.name}: text_prompts order differs from registry") + for det in dets: + total_dets += 1 + cid = det.get("class_id") + if cid is None or int(cid) < 0 or int(cid) >= len(expected_names): + errors.append(f"{p.name}: invalid class_id {cid}") + if det.get("fit_ok"): + fit_ok += 1 + + stats = { + "quaternion_files": len(files), + "files_with_detections": files_with_dets, + "detections": total_dets, + "fit_ok_ratio": fit_ok / max(total_dets, 1), + "has_k_ratio": has_k / max(files_with_dets, 1), + } + + if files_with_dets == 0: + errors.append("no quaternion json with detections") + + calib_dir = batch_dir / "calib" + if calib_dir.is_dir() and list(calib_dir.glob("*.yaml")): + if files_with_dets > 0 and has_k < files_with_dets: + errors.append(f"calib present but only {has_k}/{files_with_dets} annotated json have K") + if not allow_partial_3d and total_dets > 0: + ratio = fit_ok / total_dets + if ratio < min_fit_ratio: + errors.append( + f"fit_ok ratio {ratio:.2f} < {min_fit_ratio} (use allow_partial_3d for pilot)" + ) + + return errors, warnings, stats diff --git a/platform/as_platform/data/promote/validate/dms_yolo.py b/platform/as_platform/data/promote/validate/dms_yolo.py new file mode 100644 index 0000000..1786ac6 --- /dev/null +++ b/platform/as_platform/data/promote/validate/dms_yolo.py @@ -0,0 +1,18 @@ +"""DMS YOLO batch validation wrapper.""" +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +from as_platform.config import WORKSPACE + + +def validate_dms_task(task: str | None) -> list[str]: + cmd = [sys.executable, str(WORKSPACE / "scripts" / "validate_dms_tasks.py")] + if task: + cmd.extend(["--task", task]) + proc = subprocess.run(cmd, capture_output=True, text=True) + if proc.returncode != 0: + return [proc.stderr or proc.stdout or f"validate_dms_tasks failed exit {proc.returncode}"] + return [] diff --git a/platform/as_platform/jobs/queue.py b/platform/as_platform/jobs/queue.py index 0629f64..1f533ff 100644 --- a/platform/as_platform/jobs/queue.py +++ b/platform/as_platform/jobs/queue.py @@ -140,6 +140,10 @@ def _run_job(job_id: str) -> None: from as_platform.labeling.batch_stage import on_labeling_export_job_succeeded on_labeling_export_job_succeeded(job) + elif job.get("action") in ("build_dms", "build_adas", "build_lane"): + from as_platform.labeling.batch_stage import on_build_job_succeeded + + on_build_job_succeeded(job) except Exception as e: _patch(job_id, status="failed", finished_at=_now(), result={"ok": False, "error": str(e)}) publish("job.failed", {"job_id": job_id, "error": str(e)}) diff --git a/platform/as_platform/jobs/runner.py b/platform/as_platform/jobs/runner.py index 418f02c..ea7775e 100644 --- a/platform/as_platform/jobs/runner.py +++ b/platform/as_platform/jobs/runner.py @@ -4,6 +4,7 @@ from __future__ import annotations import json import subprocess import sys +from pathlib import Path from typing import Any from as_platform.config import WORKSPACE, PLATFORM_DIR, LANE_DATA_VIZ_ENABLED @@ -73,24 +74,52 @@ def execute_action(action: str, params: dict[str, Any]) -> dict[str, Any]: return _run_ml(["train", "lane"], timeout=86400) if action == "build_dms": - argv = ["build", "dms", p["task"]] - if p.get("pack"): - argv.extend(["--pack", str(p["pack"])]) - if p.get("batch"): - argv.extend(["--batch", str(p["batch"])]) - if p.get("all_sources"): - argv.append("--all-sources") - if p.get("dry_run"): - argv.append("--dry-run") - if p.get("skip_validate"): - argv.append("--skip-validate") - if p.get("no_refresh"): - argv.append("--no-refresh") - result = _run_ml(argv) - # 自动创建数据集快照 - _auto_snapshot("dms", task=p.get("task", "")) + from as_platform.data.promote.runner import promote_batch + + result = promote_batch( + "dms", + task=p["task"], + batch=p.get("batch"), + pack=p.get("pack"), + dry_run=bool(p.get("dry_run")), + skip_validate=bool(p.get("skip_validate")), + refresh=not p.get("no_refresh"), + ) return result + if action == "build_adas": + from as_platform.data.promote.runner import promote_batch + + return promote_batch( + "adas", + task=p.get("task", "cuboid_7cls"), + batch=p.get("batch"), + pack=p.get("pack", "adas_moon3d_v1"), + dry_run=bool(p.get("dry_run")), + skip_validate=bool(p.get("skip_validate")), + allow_partial_3d=bool(p.get("allow_partial_3d", True)), + ) + + if action == "cuboid_fit_3d": + from as_platform.db.engine import session_scope + from as_platform.db.models import LabelingCampaign + from as_platform.labeling.annotate import resolve_campaign_batch_dir + from as_platform.labeling.fit_cuboid_batch import fit_batch + + campaign_id = p.get("campaign_id", "") + batch_dir = None + if campaign_id: + with session_scope() as db: + camp = db.get(LabelingCampaign, campaign_id) + if camp: + batch_dir = resolve_campaign_batch_dir(camp) + if batch_dir is None and p.get("batch_dir"): + batch_dir = Path(p["batch_dir"]) + if batch_dir is None: + raise ValueError("cuboid_fit_3d 需要 campaign_id 或 batch_dir") + conv = fit_batch(batch_dir) + return {"ok": True, "stdout": json.dumps(conv, ensure_ascii=False), "stderr": "", "fit_convert": conv} + if action == "build_lane": result = _run_ml(["build", "lane"]) _auto_snapshot("lane") @@ -229,10 +258,7 @@ def execute_action(action: str, params: dict[str, Any]) -> dict[str, Any]: "export_ls_to_yolo: 无有效标注可导出 (written=0); " f"skipped_empty={conv.get('skipped_empty')} missing_ann={conv.get('missing_ann')}" ) - argv = ["build", "dms", task, "--pack", pack, "--batch", batch] - result = _run_ml(argv) - result["export_convert"] = conv - return result + return {"ok": True, "stdout": json.dumps(conv, ensure_ascii=False), "stderr": "", "export_convert": conv} if row.get("project") == "lane" and export == "lane_gt_txt": scripts_dir = WORKSPACE / "datasets" / "lane" / "scripts" if str(scripts_dir) not in sys.path: @@ -250,10 +276,22 @@ def execute_action(action: str, params: dict[str, Any]) -> dict[str, Any]: "export_ls_to_lane_gt: 无有效标注可导出 (written=0); " f"skipped_empty={conv.get('skipped_empty')} missing_ann={conv.get('missing_ann')}" ) - argv = ["build", "lane"] - result = _run_ml(argv) - result["export_convert"] = conv - return result + return {"ok": True, "stdout": json.dumps(conv, ensure_ascii=False), "stderr": "", "export_convert": conv} + if row.get("project") == "adas" and export == "cvat_cuboid": + from as_platform.labeling.export_cuboid_batch import export_batch as export_cuboid_batch + + with session_scope() as db: + camp = db.get(LabelingCampaign, campaign_id) + if not camp: + raise ValueError("campaign not found") + batch_dir = resolve_campaign_batch_dir(camp) + conv = export_cuboid_batch(batch_dir) + if conv.get("written", 0) == 0: + raise ValueError( + "export_cuboid_batch: 无有效 cuboid 可导出 (written=0); " + f"skipped_empty={conv.get('skipped_empty')} missing_ann={conv.get('missing_ann')}" + ) + return {"ok": True, "stdout": json.dumps(conv, ensure_ascii=False), "stderr": "", "export_convert": conv} return { "ok": True, "stdout": json.dumps({"export": export, "campaign": row}, ensure_ascii=False), diff --git a/platform/as_platform/labeling/batch_stage.py b/platform/as_platform/labeling/batch_stage.py index 0ee4592..6cd7fab 100644 --- a/platform/as_platform/labeling/batch_stage.py +++ b/platform/as_platform/labeling/batch_stage.py @@ -1,6 +1,7 @@ """同步 inbox/sources 批次 batch.meta.yaml 的 stage，与 Campaign 状态一致。""" from __future__ import annotations +import json from pathlib import Path from as_platform.data.batch import read_meta, write_meta @@ -18,6 +19,22 @@ def batch_has_yolo_labels(batch_dir: Path) -> bool: return False +def batch_has_cuboid_labels(batch_dir: Path) -> bool: + """批次是否已有导出的 ADAS quaternion_json（含非空 detections）。""" + qdir = batch_dir / "labels" / "quaternion_json" + if not qdir.is_dir(): + return False + for p in qdir.glob("*.json"): + try: + data = json.loads(p.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + continue + dets = data.get("detections") or [] + if isinstance(dets, list) and len(dets) > 0: + return True + return False + + def batch_has_lane_labels(batch_dir: Path) -> bool: """批次是否已有 UFLD mask 清单（list/train_gt.txt + annotations/*.png）。""" list_path = batch_dir / "list" / "train_gt.txt" @@ -29,6 +46,14 @@ def batch_has_lane_labels(batch_dir: Path) -> bool: return any(ann_dir.rglob("*.png")) +def _batch_has_export_labels(project: str, batch_dir: Path) -> bool: + if project == "lane": + return batch_has_lane_labels(batch_dir) + if project == "adas": + return batch_has_cuboid_labels(batch_dir) + return batch_has_yolo_labels(batch_dir) + + def update_campaign_batch_meta_stage(camp: LabelingCampaign, stage: str) -> bool: try: batch_dir = resolve_campaign_batch_dir(camp) @@ -56,8 +81,23 @@ def update_campaign_batch_meta_stage_by_id(campaign_id: str, stage: str) -> bool return update_campaign_batch_meta_stage(camp, stage) +def _advance_campaign_stage(campaign_id: str, stage: str) -> None: + with session_scope() as db: + camp = db.get(LabelingCampaign, str(campaign_id)) + if not camp: + return + camp.status = stage + db.flush() + update_campaign_batch_meta_stage(camp, stage) + + +def _batch_has_calib(batch_dir: Path) -> bool: + calib = batch_dir / "calib" + return calib.is_dir() and bool(list(calib.glob("*.yaml")) + list(calib.glob("*.yml"))) + + def on_labeling_export_job_succeeded(job: dict) -> None: - """导出 Job 成功且批次已有训练标签时进入 returned（待入库）。""" + """导出 Job 成功且批次已有训练标签时进入 returned（待 build）。""" if job.get("action") != "labeling_export": return params = job.get("params") or {} @@ -72,10 +112,46 @@ def on_labeling_export_job_succeeded(job: dict) -> None: batch_dir = resolve_campaign_batch_dir(camp) except Exception: return - has_labels = ( - batch_has_lane_labels(batch_dir) - if camp.project == "lane" - else batch_has_yolo_labels(batch_dir) - ) - if has_labels: - update_campaign_batch_meta_stage_by_id(str(cid), "returned") + project = camp.project or "dms" + if _batch_has_export_labels(project, batch_dir): + _advance_campaign_stage(str(cid), "returned") + if project == "adas" and _batch_has_calib(batch_dir): + from as_platform.jobs.queue import enqueue_job + + enqueue_job( + "cuboid_fit_3d", + {"campaign_id": str(cid)}, + async_run=True, + ) + + +def on_build_job_succeeded(job: dict) -> None: + """build Job 成功后将批次晋升 ingested。""" + action = job.get("action") + if action not in ("build_dms", "build_adas", "build_lane"): + return + params = job.get("params") or {} + batch = params.get("batch") + if not batch: + return + project = params.get("project") + if not project: + if action == "build_adas": + project = "adas" + elif action == "build_lane": + project = "lane" + else: + project = "dms" + task = params.get("task") + with session_scope() as db: + q = db.query(LabelingCampaign).filter(LabelingCampaign.batch == str(batch)) + if task: + q = q.filter(LabelingCampaign.task == str(task)) + if project: + q = q.filter(LabelingCampaign.project == str(project)) + camp = q.order_by(LabelingCampaign.created_at.desc()).first() + if not camp: + return + camp.status = "ingested" + db.flush() + update_campaign_batch_meta_stage(camp, "ingested") diff --git a/platform/as_platform/labeling/class_map.py b/platform/as_platform/labeling/class_map.py new file mode 100644 index 0000000..7c0b24e --- /dev/null +++ b/platform/as_platform/labeling/class_map.py @@ -0,0 +1,74 @@ +"""ADAS class_id 映射（BK2/MOON 单源）。""" +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import yaml + +from as_platform.config import WORKSPACE + +_ADAS_REGISTRY = WORKSPACE / "datasets" / "adas" / "adas.registry.yaml" +_LABELING_REGISTRY = WORKSPACE / "datasets" / "labeling.registry.yaml" + + +def load_adas_class_names() -> list[str]: + if _ADAS_REGISTRY.is_file(): + reg = yaml.safe_load(_ADAS_REGISTRY.read_text(encoding="utf-8")) or {} + names = (reg.get("classes") or {}).get("names") + if names: + return [str(n) for n in names] + if _LABELING_REGISTRY.is_file(): + reg = yaml.safe_load(_LABELING_REGISTRY.read_text(encoding="utf-8")) or {} + labels = (reg.get("profiles") or {}).get("cuboid_7cls", {}).get("cvat_labels") + if labels: + return [str(n) for n in labels] + from as_platform.labeling.format_converter import CUBOID_7CLS_NAMES + + return list(CUBOID_7CLS_NAMES) + + +def class_name_to_id(name: str, class_map: dict[str, int] | None = None) -> int | None: + cmap = class_map or {n: i for i, n in enumerate(load_adas_class_names())} + if name in cmap: + return cmap[name] + low = name.lower() + for k, v in cmap.items(): + if k.lower() == low: + return v + return None + + +def build_class_map(names: list[str] | None = None) -> dict[str, int]: + return {str(n): idx for idx, n in enumerate(names or load_adas_class_names())} + + +def remap_class_id(old_names: list[str], new_names: list[str], class_id: int) -> int: + if class_id < 0 or class_id >= len(old_names): + return class_id + label = old_names[class_id] + new_id = build_class_map(new_names).get(label) + if new_id is None: + for k, v in build_class_map(new_names).items(): + if k.lower() == label.lower(): + return v + return new_id if new_id is not None else class_id + + +def normalize_detection_class(det: dict[str, Any], class_map: dict[str, int] | None = None) -> dict[str, Any]: + cmap = class_map or build_class_map() + name = str(det.get("class_name") or "") + cid = det.get("class_id") + if name: + mapped = class_name_to_id(name, cmap) + if mapped is not None: + det = dict(det) + det["class_id"] = mapped + det["class_name"] = name + elif cid is not None: + names = list(cmap.keys()) + idx = int(cid) + if 0 <= idx < len(names): + det = dict(det) + det["class_name"] = names[idx] + return det diff --git a/platform/as_platform/labeling/export_cuboid_batch.py b/platform/as_platform/labeling/export_cuboid_batch.py new file mode 100644 index 0000000..767bb1d --- /dev/null +++ b/platform/as_platform/labeling/export_cuboid_batch.py @@ -0,0 +1,174 @@ +"""ls_annotations cuboid → labels/quaternion_json/*.json（ADAS MOON-3D 兼容格式）。""" +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +import yaml + +from as_platform.labeling.class_map import build_class_map, load_adas_class_names +from as_platform.labeling.format_converter import cuboid_item_to_detection + +IMG_EXTS = {".jpg", ".jpeg", ".png", ".bmp", ".webp", ".JPG", ".JPEG", ".PNG"} +ANNOTATIONS_DIRNAME = "ls_annotations" + + +def _load_cuboid_class_map() -> dict[str, int]: + return build_class_map(load_adas_class_names()) + + +def _task_id_for_image(image_path: Path, batch_dir: Path) -> str: + try: + rel = image_path.relative_to(batch_dir) + stem = rel.as_posix() + except ValueError: + stem = image_path.name + return hashlib.sha256(stem.encode()).hexdigest()[:16] + + +def _iter_batch_images(batch_dir: Path) -> list[Path]: + if not batch_dir.is_dir(): + return [] + candidates: list[Path] = [] + search_roots = [batch_dir / "images", batch_dir / "images" / "train", batch_dir] + seen: set[str] = set() + for root in search_roots: + if not root.is_dir(): + continue + for p in sorted(root.rglob("*")): + if not p.is_file() or p.suffix not in IMG_EXTS: + continue + key = str(p.resolve()) + if key in seen: + continue + seen.add(key) + candidates.append(p.resolve()) + return candidates + + +def _extract_result_regions(data: dict[str, Any]) -> list[dict[str, Any]]: + result = data.get("result") + if isinstance(result, list) and result: + return result + annotations = data.get("annotations") + if isinstance(annotations, list) and annotations: + first = annotations[0] + if isinstance(first, dict) and isinstance(first.get("result"), list): + return first["result"] + return [] + + +def _find_calib(batch_dir: Path) -> tuple[Path | None, list[list[float]] | None, list[int] | None]: + calib_dir = batch_dir / "calib" + if not calib_dir.is_dir(): + return None, None, None + yaml_files = sorted(calib_dir.glob("*.yaml")) + sorted(calib_dir.glob("*.yml")) + if not yaml_files: + return None, None, None + path = yaml_files[0] + try: + data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + except (OSError, yaml.YAMLError): + return path, None, None + K = data.get("K") + image_size = data.get("image_size") + if K and isinstance(K, list) and len(K) == 3: + return path, K, list(image_size) if image_size else None + fx = data.get("fx") + fy = data.get("fy") + cx = data.get("cx") + cy = data.get("cy") + if fx is not None and fy is not None and cx is not None and cy is not None: + K = [[float(fx), 0.0, float(cx)], [0.0, float(fy), float(cy)], [0.0, 0.0, 1.0]] + return path, K, list(image_size) if image_size else None + return path, None, list(image_size) if image_size else None + + +def _resolve_image_for_ann(data: dict[str, Any], batch_dir: Path, task_id: str) -> Path | None: + image_name = data.get("image") + if image_name: + for root in (batch_dir / "images", batch_dir): + candidate = root / str(image_name) + if candidate.is_file(): + return candidate + for p in root.rglob(str(image_name)): + if p.is_file(): + return p + for image_path in _iter_batch_images(batch_dir): + if _task_id_for_image(image_path, batch_dir) == task_id: + return image_path + return None + + +def export_batch(batch_dir: Path) -> dict[str, Any]: + """导出 cuboid ls_annotations → quaternion_json。""" + batch_dir = batch_dir.resolve() + class_map = _load_cuboid_class_map() + calib_path, K, calib_size = _find_calib(batch_dir) + ann_dir = batch_dir / "labels" / ANNOTATIONS_DIRNAME + out_dir = batch_dir / "labels" / "quaternion_json" + out_dir.mkdir(parents=True, exist_ok=True) + + written = 0 + skipped_empty = 0 + missing_ann = 0 + + for ann_path in sorted(ann_dir.glob("*.json")): + task_id = ann_path.stem + try: + data = json.loads(ann_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + missing_ann += 1 + continue + regions = _extract_result_regions(data) + cuboids = [r for r in regions if r.get("type") == "cuboid"] + if not cuboids: + skipped_empty += 1 + continue + + image_path = _resolve_image_for_ann(data, batch_dir, task_id) + if not image_path: + missing_ann += 1 + continue + + detections: list[dict[str, Any]] = [] + for item in cuboids: + det = cuboid_item_to_detection(item, class_map, K=K) + if det: + detections.append(det) + if not detections: + skipped_empty += 1 + continue + + img_w = int((cuboids[0].get("original_width") or (calib_size or [1920, 1080])[0])) + img_h = int((cuboids[0].get("original_height") or (calib_size or [1920, 1080])[1])) + + payload: dict[str, Any] = { + "image": str(image_path), + "image_stem": image_path.stem, + "image_size": [img_w, img_h], + "coordinate_frame": "opencv_camera", + "boxes3d_format": "center_3d + dimensions_wlh + quaternion_wxyz", + "text_prompts": load_adas_class_names(), + "num_detections": len(detections), + "detections": detections, + } + if K: + payload["K"] = K + payload["k_source"] = calib_path.name if calib_path else "fixed_calib" + else: + payload["k_source"] = "missing_calib" + + out_path = out_dir / f"{image_path.stem}.json" + out_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") + written += 1 + + return { + "written": written, + "skipped_empty": skipped_empty, + "missing_ann": missing_ann, + "missing_calib": calib_path is None or K is None, + "calib": str(calib_path) if calib_path else None, + } diff --git a/platform/as_platform/labeling/fit_cuboid_batch.py b/platform/as_platform/labeling/fit_cuboid_batch.py new file mode 100644 index 0000000..c75bbd3 --- /dev/null +++ b/platform/as_platform/labeling/fit_cuboid_batch.py @@ -0,0 +1,95 @@ +"""Batch-level cuboid 3D fit for quaternion_json.""" +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +IMG_EXTS = {".jpg", ".jpeg", ".png", ".bmp", ".webp"} + + +def _task_id_for_image(image_path: Path, batch_dir: Path) -> str: + try: + rel = image_path.relative_to(batch_dir) + stem = rel.as_posix() + except ValueError: + stem = image_path.name + return hashlib.sha256(stem.encode()).hexdigest()[:16] + + +def _load_ls_cuboid_points(batch_dir: Path, stem: str) -> list[list[float]]: + ann_dir = batch_dir / "labels" / "ls_annotations" + if not ann_dir.is_dir(): + return [] + for p in ann_dir.glob("*.json"): + try: + data = json.loads(p.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + continue + img = str(data.get("image") or "") + if stem in img or p.stem: + regions = data.get("result") or [] + pts_list = [] + for r in regions: + if r.get("type") != "cuboid": + continue + pts = list(r.get("points") or []) + if len(pts) >= 16: + pts_list.append(pts[:16]) + if pts_list: + return pts_list + return [] + + +def fit_batch(batch_dir: Path) -> dict[str, Any]: + from algorithms.adas_mono3d.fit_cuboid import fit_cuboid_detection + + batch_dir = batch_dir.resolve() + qdir = batch_dir / "labels" / "quaternion_json" + if not qdir.is_dir(): + raise ValueError(f"missing {qdir}") + + updated = 0 + fit_ok = 0 + total = 0 + for p in sorted(qdir.glob("*.json")): + data = json.loads(p.read_text(encoding="utf-8")) + K = data.get("K") + if not K: + continue + stem = data.get("image_stem") or p.stem + cuboid_pts_list = _load_ls_cuboid_points(batch_dir, stem) + new_dets = [] + for i, det in enumerate(data.get("detections") or []): + det = dict(det) + if det.get("fit_ok"): + new_dets.append(det) + total += 1 + fit_ok += 1 + continue + class_name = str(det.get("class_name") or "car") + points = cuboid_pts_list[i] if i < len(cuboid_pts_list) else None + if not points: + box = det.get("box2d_xyxy") or [] + if len(box) >= 4: + x1, y1, x2, y2 = box[:4] + points = [x1, y1, x2, y1, x1, y2, x2, y2, x1, y1, x2, y1, x1, y2, x2, y2] + if points: + fitted = fit_cuboid_detection(points, K, class_name) + det.update({k: v for k, v in fitted.items() if k != "box2d_xyxy" or "box2d_xyxy" not in det}) + new_dets.append(det) + total += 1 + if det.get("fit_ok"): + fit_ok += 1 + data["detections"] = new_dets + data["num_detections"] = len(new_dets) + p.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8") + updated += 1 + + return { + "updated_files": updated, + "detections": total, + "fit_ok": fit_ok, + "fit_ok_ratio": fit_ok / max(total, 1), + } diff --git a/platform/as_platform/labeling/format_converter.py b/platform/as_platform/labeling/format_converter.py index e013e7c..2be4328 100644 --- a/platform/as_platform/labeling/format_converter.py +++ b/platform/as_platform/labeling/format_converter.py @@ -491,6 +491,71 @@ def convert_cvat_kitti_export_to_hsap(kitti_data: bytes, output_dir: Path) -> in return count +# ═══════════════════════════════════════════════════════ +# CVAT cuboid 16pt → HSAP quaternion_json detection (MVP) +# ═══════════════════════════════════════════════════════ + +CUBOID_7CLS_NAMES = [ + "pedestrian", + "car", + "truck", + "bus", + "motorcycle", + "tricycle", + "traffic cone", +] + + +def cuboid_points_to_box2d(points: list[float]) -> list[float] | None: + """从 CVAT cuboid 16 点（8 个 x,y 对）计算 axis-aligned 2D bbox。""" + if len(points) < 16: + return None + xs = [float(points[i]) for i in range(0, 16, 2)] + ys = [float(points[i]) for i in range(1, 16, 2)] + return [min(xs), min(ys), max(xs), max(ys)] + + +def cuboid_item_to_detection( + item: dict[str, Any], + class_map: dict[str, int], + *, + K: list[list[float]] | None = None, +) -> dict[str, Any] | None: + """ls_annotations cuboid 条目 → quaternion_json detection（MVP：2D bbox + 可选 3D 占位）。""" + label = str(item.get("label") or "") + class_id = class_map.get(label) + if class_id is None: + for name, cid in class_map.items(): + if name.lower() == label.lower(): + class_id = cid + break + if class_id is None: + return None + + points = item.get("points") or [] + if len(points) < 16: + for key in ( + "xtl1", "ytl1", "xtr1", "ytr1", "xbl1", "ybl1", "xbr1", "ybr1", + "xtl2", "ytl2", "xtr2", "ytr2", "xbl2", "ybl2", "xbr2", "ybr2", + ): + if key in item: + points.append(float(item[key])) + box2d = cuboid_points_to_box2d(points) + if not box2d: + return None + + det: dict[str, Any] = { + "class_id": class_id, + "class_name": label, + "score": 1.0, + "box2d_xyxy": box2d, + "fit_ok": False, + } + if K: + det["K_used"] = True + return det + + # ═══════════════════════════════════════════════════════ # ADAS 3D Quaternion JSON → CVAT cuboid XML # ═══════════════════════════════════════════════════════ diff --git a/platform/as_platform/labeling/service.py b/platform/as_platform/labeling/service.py index 735cdfb..3eadded 100644 --- a/platform/as_platform/labeling/service.py +++ b/platform/as_platform/labeling/service.py @@ -18,6 +18,7 @@ from as_platform.labeling.batch_stage import ( on_labeling_export_job_succeeded, update_campaign_batch_meta_stage, ) +from as_platform.labeling.stage import effective_stage, matches_stage_filter from as_platform.labeling.scope import ( enrich_batch_labels, format_scope_key, @@ -120,11 +121,14 @@ def list_labeling_batches( def _append(b: dict[str, Any]) -> None: if b.get("registry_only"): return - if stage and b.get("stage") != stage: + raw_stage = b.get("stage") + eff = effective_stage(raw_stage) + if stage and not matches_stage_filter(raw_stage, stage): return - if b.get("stage") not in allowed_stages: + if eff not in allowed_stages and raw_stage not in allowed_stages: return row = enrich_batch_labels(b, reg) + row["stage"] = eff or raw_stage cid = _campaign_id( row["project"], row.get("task") or "", row.get("mode"), row["batch"], row.get("location") or "inbox" ) @@ -470,6 +474,48 @@ def trigger_labeling_export(campaign_id: str) -> dict[str, Any]: return {"ok": True, "job": job, "export_job": ej, "export_default": row.get("export_default")} +def get_batch_export_stats(campaign_id: str) -> dict[str, Any]: + from as_platform.labeling.annotate import resolve_campaign_batch_dir + from as_platform.data.promote.validate.adas_cuboid import validate_adas_cuboid_batch + from as_platform.labeling.batch_stage import batch_has_cuboid_labels, batch_has_yolo_labels + + with session_scope() as db: + camp = db.get(LabelingCampaign, campaign_id) + if not camp: + raise FileNotFoundError("campaign not found") + project = camp.project + batch_dir = resolve_campaign_batch_dir(camp) + if project == "adas": + _errors, warnings, stats = validate_adas_cuboid_batch(batch_dir, allow_partial_3d=True) + calib = (batch_dir / "calib").is_dir() and bool(list((batch_dir / "calib").glob("*.yaml"))) + return { + "project": "adas", + "campaign_id": campaign_id, + "pack_default": "adas_moon3d_v1", + "quaternion_files": stats.get("quaternion_files", 0), + "fit_ok_ratio": stats.get("fit_ok_ratio", 0), + "missing_calib": not calib, + "stats": stats, + "warnings": warnings, + } + return { + "project": project, + "campaign_id": campaign_id, + "has_yolo": batch_has_yolo_labels(batch_dir), + "has_cuboid": batch_has_cuboid_labels(batch_dir), + } + + +def trigger_cuboid_fit(campaign_id: str) -> dict[str, Any]: + row = get_campaign(campaign_id) + if not row: + raise FileNotFoundError("campaign not found") + if row.get("project") != "adas": + raise ValueError("cuboid_fit_3d 仅适用于 ADAS") + job = enqueue_job("cuboid_fit_3d", {"campaign_id": campaign_id}, async_run=True) + return {"ok": True, "job": job} + + # ═══════════════════════════════════════════════════════ # CVAT 集成辅助 # ═══════════════════════════════════════════════════════ diff --git a/platform/as_platform/labeling/stage.py b/platform/as_platform/labeling/stage.py new file mode 100644 index 0000000..feddefc --- /dev/null +++ b/platform/as_platform/labeling/stage.py @@ -0,0 +1,29 @@ +"""标注批次 stage 读时归一化（兼容旧 pipeline）。""" +from __future__ import annotations + +STAGE_ALIASES: dict[str, str] = { + "review_approved": "labeling_submitted", +} + +CANONICAL_STAGES = ( + "raw_pool", + "out_for_labeling", + "in_review", + "review_rejected", + "labeling_submitted", + "returned", + "ingested", +) + + +def effective_stage(stage: str | None) -> str | None: + if not stage: + return stage + return STAGE_ALIASES.get(stage, stage) + + +def matches_stage_filter(batch_stage: str | None, filter_stage: str | None) -> bool: + if not filter_stage: + return True + eff = effective_stage(batch_stage) + return eff == filter_stage or batch_stage == filter_stage diff --git a/platform/as_platform/tests/test_unified_ingest_sdk.py b/platform/as_platform/tests/test_unified_ingest_sdk.py new file mode 100644 index 0000000..010bf32 --- /dev/null +++ b/platform/as_platform/tests/test_unified_ingest_sdk.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python3 +"""Unified Ingest SDK 单元测试（无 pytest 依赖）。""" +from __future__ import annotations + +import json +import shutil +import sys +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +PLATFORM = ROOT / "platform" +if str(PLATFORM) not in sys.path: + sys.path.insert(0, str(PLATFORM)) +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + + +def test_stage_aliases() -> None: + from as_platform.labeling.stage import effective_stage, matches_stage_filter + + assert effective_stage("review_approved") == "labeling_submitted" + assert effective_stage("returned") == "returned" + assert matches_stage_filter("review_approved", "labeling_submitted") + assert not matches_stage_filter("raw_pool", "returned") + + +def test_bk2_class_map() -> None: + from as_platform.labeling.class_map import ( + build_class_map, + load_adas_class_names, + normalize_detection_class, + remap_class_id, + ) + + names = load_adas_class_names() + assert names[0] == "pedestrian" + assert names[1] == "car" + cmap = build_class_map(names) + assert cmap["car"] == 1 + assert cmap["pedestrian"] == 0 + + old = ["car", "pedestrian", "truck", "bus", "motorcycle", "tricycle", "traffic cone"] + assert remap_class_id(old, names, 0) == 1 # car was 0, now 1 + + det = normalize_detection_class({"class_name": "car", "class_id": 99}) + assert det["class_id"] == 1 + + +def test_validate_adas_cuboid() -> None: + from as_platform.data.promote.validate.adas_cuboid import validate_adas_cuboid_batch + from as_platform.labeling.class_map import load_adas_class_names + + with tempfile.TemporaryDirectory() as td: + batch = Path(td) + qdir = batch / "labels" / "quaternion_json" + qdir.mkdir(parents=True) + names = load_adas_class_names() + good = { + "detections": [{"class_id": 1, "class_name": "car", "fit_ok": False}], + "text_prompts": names, + "K": [[1000, 0, 960], [0, 1000, 540], [0, 0, 1]], + } + (qdir / "a.json").write_text(json.dumps(good), encoding="utf-8") + (qdir / "empty.json").write_text(json.dumps({"detections": []}), encoding="utf-8") + (batch / "calib").mkdir() + (batch / "calib" / "cam.yaml").write_text("K: []\n", encoding="utf-8") + + errors, warnings, stats = validate_adas_cuboid_batch(batch, allow_partial_3d=True) + assert not errors, errors + assert stats["files_with_detections"] == 1 + assert any("empty" in w for w in warnings) + + +def test_fit_cuboid_detection() -> None: + from algorithms.adas_mono3d.fit_cuboid import cuboid_points_to_box2d, fit_cuboid_detection + + pts = [770.0, 347.0, 834.0, 347.0, 772.0, 423.0, 835.0, 423.0, + 806.0, 357.0, 861.0, 357.0, 807.0, 422.0, 862.0, 422.0] + box = cuboid_points_to_box2d(pts) + assert box is not None + assert box[0] < box[2] and box[1] < box[3] + + K = [[1189.7, 0, 1007.5], [0, 1189.7, 517.5], [0, 0, 1]] + out = fit_cuboid_detection(pts, K, "car") + assert "center_3d" in out + assert "dimensions_wlh" in out + assert "quaternion_wxyz" in out + assert len(out["quaternion_wxyz"]) == 4 + + +def test_export_cuboid_batch_class_id() -> None: + from as_platform.labeling.export_cuboid_batch import export_batch + + with tempfile.TemporaryDirectory() as td: + batch = Path(td) + (batch / "images").mkdir() + img = batch / "images" / "frame1.jpg" + img.write_bytes(b"\xff\xd8\xff") + calib = batch / "calib" / "cam0.yaml" + calib.parent.mkdir() + calib.write_text( + "K:\n - [1000, 0, 960]\n - [0, 1000, 540]\n - [0, 0, 1]\n", + encoding="utf-8", + ) + ann_dir = batch / "labels" / "ls_annotations" + ann_dir.mkdir(parents=True) + ann = { + "image": "frame1.jpg", + "result": [{ + "type": "cuboid", + "label": "car", + "points": [770.0, 347.0, 834.0, 347.0, 772.0, 423.0, 835.0, 423.0, + 806.0, 357.0, 861.0, 357.0, 807.0, 422.0, 862.0, 422.0], + "original_width": 1920, + "original_height": 1080, + }], + } + import hashlib + tid = hashlib.sha256(b"images/frame1.jpg").hexdigest()[:16] + (ann_dir / f"{tid}.json").write_text(json.dumps(ann), encoding="utf-8") + + result = export_batch(batch) + assert result["written"] == 1 + qjson = batch / "labels" / "quaternion_json" / "frame1.json" + assert qjson.is_file() + data = json.loads(qjson.read_text()) + assert data["text_prompts"][0] == "pedestrian" + assert data["detections"][0]["class_id"] == 1 + assert data["detections"][0]["class_name"] == "car" + + +def test_refresh_adas_lists() -> None: + from as_platform.data.promote.manifest import refresh_adas_lists + + with tempfile.TemporaryDirectory() as td: + pack_root = Path(td) / "packs" / "test_pack" + src = pack_root / "sources" / "batch_a" / "labels" / "quaternion_json" + src.mkdir(parents=True) + (src / "img1.json").write_text('{"detections":[{}]}', encoding="utf-8") + (src / "img2.json").write_text('{"detections":[{}]}', encoding="utf-8") + + wf = { + "projects": { + "adas": { + "root": str(Path(td)), + "registry": "adas.registry.yaml", + } + } + } + (Path(td) / "adas.registry.yaml").write_text("split:\n val_ratio: 0.5\n", encoding="utf-8") + + out = refresh_adas_lists(wf, pack="test_pack") + train = Path(out["train_list"]).read_text().strip().splitlines() + val = Path(out["val_list"]).read_text().strip().splitlines() + assert len(train) + len(val) == 2 + assert Path(out["pack_index"]).is_file() + + +def test_promote_adas_dry_run() -> None: + from as_platform.data.promote.adas_cuboid import AdasCuboidPromoteAdapter + from as_platform.data.promote.base import PromoteContext + from as_platform.labeling.class_map import load_adas_class_names + + with tempfile.TemporaryDirectory() as td: + batch = Path(td) / "inbox" / "cuboid_7cls" / "b1" + qdir = batch / "labels" / "quaternion_json" + qdir.mkdir(parents=True) + names = load_adas_class_names() + payload = { + "detections": [{"class_id": 1, "class_name": "car"}], + "text_prompts": names, + "K": [[1000, 0, 960], [0, 1000, 540], [0, 0, 1]], + } + (qdir / "f.json").write_text(json.dumps(payload), encoding="utf-8") + (batch / "images").mkdir() + (batch / "images" / "f.jpg").write_bytes(b"x") + + root = Path(td) + ctx = PromoteContext( + project="adas", + task="cuboid_7cls", + batch="b1", + pack="test_pack", + batch_dir=batch, + project_root=root, + dry_run=True, + ) + adapter = AdasCuboidPromoteAdapter() + assert adapter.validate(ctx) == [] + result = adapter.promote(ctx) + assert result.ok + assert result.detail.get("dry_run") is True + + +def main() -> None: + tests = [ + test_stage_aliases, + test_bk2_class_map, + test_validate_adas_cuboid, + test_fit_cuboid_detection, + test_export_cuboid_batch_class_id, + test_refresh_adas_lists, + test_promote_adas_dry_run, + ] + for fn in tests: + fn() + print(f"OK {fn.__name__}") + print(f"ALL {len(tests)} PASSED") + + +if __name__ == "__main__": + main() diff --git a/platform/web/src/app/hsap-api.ts b/platform/web/src/app/hsap-api.ts index bd34918..38190a6 100644 --- a/platform/web/src/app/hsap-api.ts +++ b/platform/web/src/app/hsap-api.ts @@ -162,6 +162,12 @@ export const hsapApi = { labelingExport: (campaignId: string) => postJson<{ ok: boolean; job?: { id: string } }>(`${API_BASE}/api/v1/labeling/campaigns/${campaignId}/export`), + labelingExportStats: (campaignId: string) => + fetchJson>(`${API_BASE}/api/v1/labeling/campaigns/${campaignId}/export-stats`), + + cuboidFit: (campaignId: string) => + postJson>(`${API_BASE}/api/v1/labeling/campaigns/${campaignId}/cuboid-fit`), + submitLabelingCampaign: (campaignId: string) => postJson>(`${API_BASE}/api/v1/labeling/campaigns/${campaignId}/submit`), diff --git a/platform/web/src/modules/labeling/pages/CampaignsPage.tsx b/platform/web/src/modules/labeling/pages/CampaignsPage.tsx index 1f4d4b9..2ff3199 100644 --- a/platform/web/src/modules/labeling/pages/CampaignsPage.tsx +++ b/platform/web/src/modules/labeling/pages/CampaignsPage.tsx @@ -339,7 +339,13 @@ export const CampaignsPage: React.FC = () => { @@ -347,7 +353,7 @@ export const CampaignsPage: React.FC = () => { onClick={() => handleSubmit(b.campaign_id!)} className="inline-flex items-center gap-1 px-3 py-1.5 text-xs font-medium rounded-lg bg-green-50 text-green-700 hover:bg-green-100 transition-colors" > - ✅ 提交 + ✅ 提交质检 ; })} @@ -86,33 +137,29 @@ export const ExportPage: React.FC = () => { - {/* Workflow guide */}
流程说明
1 - 标注提交 — 标注员在 Campaign 中完成标注后，点击"提交批次" + 提交质检 — 标注员完成标注后，在标注进度页点击「提交质检」
2 - - 导出标注 — 在此页面点击"执行导出"，将标注结果转为 YOLO 格式 - {hasData && （下表有待导出批次）} - + 质检通过 — 协调员在质检页审核，通过后批次进入「待导出」
3 - 供应商回标 — 如果是外部供应商标注的，点击"导入供应商"上传 ZIP 回标文件 - ZIP 格式要求：每张图片对应一个同名 .txt 标注文件（YOLO 格式），放在同一目录下打包 + 执行导出 — 将 CVAT 标注转为训练格式（DMS→YOLO，ADAS→quaternion_json） + {hasData && （下表有待处理批次）}
4 - 入库 build — 导出完成后，批次进入 待入库 状态，可通过审核队列提交 build - build 成功后自动生成数据集版本快照 + 提交 build — 导出完成后进入 待 build，在此提交 build 并经审核队列批准后变为 已入库 + 「待 build」≠「已入库」；ingested 批次不会出现在本页
@@ -127,10 +174,17 @@ export const ExportPage: React.FC = () => {
{b.batch} - {b.task || "—"} - {b.stage === "labeling_submitted" ? "待导出" : "待入库"} + {b.project}/{b.task || "—"} + {b.stage === "labeling_submitted" ? "待导出" : "待 build"}
{b.campaign_id?.slice(0, 16) || "—"}
+ {b.stage === "returned" && b.project === "adas" && b.campaign_id && statsMap[b.campaign_id] && ( +
+ quaternion: {String(statsMap[b.campaign_id].quaternion_files ?? "—")} · + fit_ok: {((Number(statsMap[b.campaign_id].fit_ok_ratio) || 0) * 100).toFixed(0)}% · + pack: adas_moon3d_v1 +
+ )}
{b.campaign_id && b.stage === "labeling_submitted" && ( @@ -139,7 +193,24 @@ export const ExportPage: React.FC = () => { )} - {b.stage === "returned" && ✓ 已入库} + {b.stage === "returned" && ( + <> + {b.project === "adas" && b.campaign_id && ( + + )} + + 审核队列 → + + )}
@@ -147,8 +218,8 @@ export const ExportPage: React.FC = () => { ) : (
-

暂无待导出或待入库的批次

-

完成标注后，在标注进度页提交批次，即可在此处导出

+

暂无待导出或待 build 的批次

+

完成标注并质检通过后，批次会出现在此处

)} diff --git a/platform/web/src/modules/labeling/pages/QualityReviewPage.tsx b/platform/web/src/modules/labeling/pages/QualityReviewPage.tsx index 5d90c34..e4a9525 100644 --- a/platform/web/src/modules/labeling/pages/QualityReviewPage.tsx +++ b/platform/web/src/modules/labeling/pages/QualityReviewPage.tsx @@ -33,7 +33,7 @@ const ReviewListPage: React.FC = () => { const results: LabelingBatchRow[] = []; const [inReview, approved, rejected] = await Promise.allSettled([ hsapApi.labelingBatches({ stage: "in_review", limit: 100 }), - hsapApi.labelingBatches({ stage: "review_approved", limit: 50 }), + hsapApi.labelingBatches({ stage: "labeling_submitted", limit: 50 }), hsapApi.labelingBatches({ stage: "review_rejected", limit: 50 }), ]); if (inReview.status === "fulfilled") results.push(...((inReview.value.items || []) as LabelingBatchRow[])); @@ -80,7 +80,7 @@ const ReviewListPage: React.FC = () => { {/* Filter chips */}
{["全部", "质检中", "已通过", "已退回"].map((label, i) => { - const val = i === 0 ? "" : ["in_review", "review_approved", "review_rejected"][i - 1]; + const val = i === 0 ? "" : ["in_review", "labeling_submitted", "review_rejected"][i - 1]; return (
{b.stage === "in_review" && } - {b.stage === "review_approved" && ✓ 已通过} + {b.stage === "labeling_submitted" && ( + + )} {b.stage === "review_rejected" && ✗ 已退回}
diff --git a/platform/web/src/modules/labeling/pages/WorkbenchPage.tsx b/platform/web/src/modules/labeling/pages/WorkbenchPage.tsx index 55757dc..4b19689 100644 --- a/platform/web/src/modules/labeling/pages/WorkbenchPage.tsx +++ b/platform/web/src/modules/labeling/pages/WorkbenchPage.tsx @@ -198,6 +198,12 @@ export const WorkbenchPage: React.FC = () => { ✏️ 进入标注 )} + {b.stage === "returned" && ( + + 🏗 提交 build + + )} diff --git a/scripts/smoke_adas_promote.sh b/scripts/smoke_adas_promote.sh new file mode 100755 index 0000000..97020de --- /dev/null +++ b/scripts/smoke_adas_promote.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# ADAS cuboid export → 3D fit → promote smoke (val_front6mm_pilot) +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +BATCH="${AS_SMOKE_BATCH:-val_front6mm_pilot}" +BATCH_DIR="${AS_SMOKE_BATCH_DIR:-$ROOT/../data/送标/adas/inbox/cuboid_7cls/$BATCH}" + +export PYTHONPATH="$ROOT/platform:$ROOT" + +python3 < 0, "export wrote 0" + +fit = fit_batch(batch_dir) +print("fit", fit) + +qfiles = list((batch_dir / "labels/quaternion_json").glob("*.json")) +q = None +for p in qfiles: + d = json.loads(p.read_text()) + if d.get("detections"): + q = p + break +assert q, "no quaternion json with detections" +data = json.loads(q.read_text()) +det = data["detections"][0] +names = load_adas_class_names() +assert names[0] == "pedestrian", names +assert det["class_name"] == "car" +assert det["class_id"] == build_class_map(names)["car"], det + +result = promote_batch( + "adas", + task="cuboid_7cls", + batch=batch_dir.name, + pack="adas_moon3d_v1", + batch_dir=batch_dir, + skip_validate=False, + allow_partial_3d=True, +) +print("promote", result) + +pack_root = Path("$ROOT/datasets/adas/packs/adas_moon3d_v1") +dest = pack_root / "sources" / batch_dir.name +assert dest.is_dir(), dest +assert (dest / "labels" / "quaternion_json").is_dir() +assert (pack_root / "lists" / "train_stems.txt").is_file() +print("SMOKE_ADAS_PROMOTE_OK") +PY diff --git a/scripts/smoke_labeling_api.sh b/scripts/smoke_labeling_api.sh index 99056a1..42513d7 100755 --- a/scripts/smoke_labeling_api.sh +++ b/scripts/smoke_labeling_api.sh @@ -10,6 +10,12 @@ python3 "$ROOT/datasets/dms/scripts/test_export_ls_to_yolo.py" echo "==> offline export_ls_to_lane_gt unit tests" python3 "$ROOT/datasets/lane/scripts/test_export_ls_to_lane_gt.py" +echo "==> offline unified ingest SDK unit tests" +PYTHONPATH="$ROOT/platform:$ROOT" python3 "$ROOT/platform/as_platform/tests/test_unified_ingest_sdk.py" + +echo "==> offline ADAS promote smoke (export/fit/promote)" +bash "$ROOT/scripts/smoke_adas_promote.sh" + if [[ "${HSAP_API_SKIP:-0}" == "1" ]]; then echo "SKIP API tests (HSAP_API_SKIP=1)" echo "OK (offline only)" diff --git a/workflow.registry.yaml b/workflow.registry.yaml index 401b416..06d1ec7 100644 --- a/workflow.registry.yaml +++ b/workflow.registry.yaml @@ -54,4 +54,10 @@ projects: root: datasets/adas # 数据湖：宿主机 DATA/data/送标/adas → inbox/{task}/{batch}/images/ registry: adas.registry.yaml - active_packs: [] + packs_registry: data_packs.yaml + base_pack: adas_moon3d_v1 + active_packs: + - adas_moon3d_v1 + merge: + train: lists/train_stems.txt + val: lists/val_stems.txt