diff --git a/.gitignore b/.gitignore index 3059b71..4019353 100644 --- a/.gitignore +++ b/.gitignore @@ -49,6 +49,8 @@ datasets/**/inbox/ datasets/**/archive/ datasets/**/sources/ datasets/**/DATASET/ +datasets/**/versions/ +datasets/**/manifests/ingest_log.jsonl datasets/**/*.jpg datasets/**/*.jpeg datasets/**/*.png diff --git a/docs/ADAS_MOON3D_PACK.md b/docs/ADAS_MOON3D_PACK.md index accb4dc..095ea2b 100644 --- a/docs/ADAS_MOON3D_PACK.md +++ b/docs/ADAS_MOON3D_PACK.md @@ -1,13 +1,18 @@ -# ADAS MOON-3D 训练包 `adas_moon3d_v1` +# ADAS 前向训练包 `adas_v1` + +> **2026-07 更新**：pack 已从 `adas_moon3d_v1` 重命名为 `adas_v1`（见 `workflow.registry.yaml`）。 +> 新增 YOLO HBB 导出路径和 2D 矩形标注支持。 ## 目录结构 ```text -datasets/adas/packs/adas_moon3d_v1/ +datasets/adas/packs/adas_v1/ ├── sources/{batch}/ │ ├── images/ │ ├── calib/ -│ └── labels/quaternion_json/ +│ └── labels/ +│ ├── quaternion_json/ # 3D cuboid 导出（有 calib 时优先） +│ └── yolo-hbb/ # 2D YOLO HBB 导出（无 calib 回退） ├── lists/ │ ├── train_stems.txt │ └── val_stems.txt @@ -32,20 +37,34 @@ datasets/adas/packs/adas_moon3d_v1/ ## 管线 1. **labeling_export** — CVAT ls_annotations → `labels/quaternion_json/*.json` + - 支持 **cuboid**（3D 框）和 **rectanglelabels**（2D 矩形）两种标注类型 + - 2D 矩形自动转为 quaternion_json detection（`fit_ok=false`, `source="2d_rect"`） 2. **cuboid_fit_3d**（有 calib 时自动触发）— 补全 3D 字段 3. **build_adas**（审核）— `promote_batch` 复制到 pack + 刷新 stem 列表 + - 优先使用 quaternion_json，无 cuboid 标注时回退到 YOLO HBB + +### 导出路径说明 + +| 标注类型 | 导出产物 | 条件 | +|---------|---------|------| +| cuboid (3D) | `labels/quaternion_json/*.json` | CVAT cuboid 标注 | +| rectanglelabels (2D) | `labels/quaternion_json/*.json` | 2D 矩形自动桥接 | +| YOLO HBB | `labels/yolo-hbb/*.txt` | 无 cuboid 时使用 `export_adas_yolo` | ## CLI ```bash -# 导出（平台 Job 或脚本） +# cuboid 导出（平台 Job） PYTHONPATH=platform python3 -c "from as_platform.labeling.export_cuboid_batch import export_batch; ..." +# YOLO HBB 导出（平台 Job，无 cuboid 时自动选择） +# 由 jobs/runner.py 调用 datasets/adas/scripts/export_ls_to_yolo.py + # 3D 拟合 PYTHONPATH=platform python3 -c "from as_platform.labeling.fit_cuboid_batch import fit_batch; ..." # 入包 -python as.py build adas cuboid_7cls --batch val_front6mm_pilot --pack adas_moon3d_v1 +python as.py build adas cuboid_7cls --batch val_front6mm_pilot --pack adas_v1 ``` ## Smoke @@ -54,7 +73,6 @@ python as.py build adas cuboid_7cls --batch val_front6mm_pilot --pack adas_moon3 bash scripts/smoke_adas_promote.sh ``` -## 与 dms/packs/adas_v1 的区别 +## 数据目录 -- `dms/packs/adas_v1`：2D YOLO 历史包（[`scripts/organize_adas.py`](../scripts/organize_adas.py)） -- `datasets/adas/packs/adas_moon3d_v1`：MOON-3D quaternion_json 3D GT +ADAS 前向数据在 **数据目录** → **前向** 视图中展示，按 pack → batch 层级显示图片数、标注数和类别分布。 diff --git a/docs/DATA_LAKE_GAP.md b/docs/DATA_LAKE_GAP.md index 7de94bd..f97eaef 100644 --- a/docs/DATA_LAKE_GAP.md +++ b/docs/DATA_LAKE_GAP.md @@ -18,7 +18,7 @@ - 数据候选：`platform/as_platform/db/models.py` → `DatasetCandidate` - 上传 API：`server.py` → `upload/file`、`inspect-upload` -- 审核：`audit/queue.py`、`/api/v1/approvals/*` +- 审核：`audit/queue.py`、`/api/v1/system/audit/*` - 入湖 CLI：`as.py build` / `add` + `ingest_incremental.py` ## 建议下一里程碑（未在本汇总 plan 全量实现） diff --git a/docs/DMS_E2E_2IMG_TEST.md b/docs/DMS_E2E_2IMG_TEST.md index a5ed37d..06b5bee 100644 --- a/docs/DMS_E2E_2IMG_TEST.md +++ b/docs/DMS_E2E_2IMG_TEST.md @@ -119,7 +119,7 @@ bash scripts/smoke_dms_e2e_2img.sh setup ### Commit `0b8ade0` — Unified Ingest SDK - `platform/as_platform/data/promote/` — DMS/ADAS promote_batch -- ADAS cuboid export / fit / `adas_moon3d_v1` +- ADAS cuboid export / fit / `adas_v1` - `platform/as_platform/tests/test_unified_ingest_sdk.py`（7 项单元测试） - `scripts/smoke_adas_promote.sh` - `scripts/smoke_labeling_api.sh` 已接入上述测试 diff --git a/docs/HANDOVER.md b/docs/HANDOVER.md index 1e5143f..fdcca72 100644 --- a/docs/HANDOVER.md +++ b/docs/HANDOVER.md @@ -350,7 +350,7 @@ python as.py train dms dam --track platform # 平台训练（需审核） | `/api/v1/system` | `system_routes.py` | 审核队列、任务监控、执行日志 | | `/api/v1/delivery` | `delivery_routes.py` | 批次交付、Bitable 集成 | | `/api/v1/feishu` | `feishu_routes.py` | Bitable Webhook 回调 | -| `/api/v1/` (根) | `server.py` | health, dashboard, catalog, approvals, simulate, agents | +| `/api/v1/` (根) | `server.py` | health, dashboard, catalog, simulate, agents | ### 6.2 核心端点速查 @@ -366,10 +366,10 @@ GET /api/v1/catalog POST /api/v1/catalog # 审核流程 -GET /api/v1/approvals -POST /api/v1/approvals/{id}/approve -POST /api/v1/approvals/{id}/reject -GET /api/v1/approvals/{id}/preview +GET /api/v1/system/audit +POST /api/v1/system/audit/{id}/approve +POST /api/v1/system/audit/{id}/reject +GET /api/v1/system/audit/{id}/preview # 任务队列 GET /api/v1/jobs diff --git a/docs/LABELING_SOP.md b/docs/LABELING_SOP.md index 86babaa..1322098 100644 --- a/docs/LABELING_SOP.md +++ b/docs/LABELING_SOP.md @@ -15,7 +15,7 @@ **协调员 QA 门禁路径（推荐）：** 标完 → **提交质检** (`in_review`) → 质检通过 (`labeling_submitted`) → **执行导出** → `returned` → **提交 build**（审核）→ `ingested`。 -ADAS cuboid 详见 [ADAS_MOON3D_PACK.md](./ADAS_MOON3D_PACK.md)：导出 → 可选 `cuboid_fit_3d` → `build_adas` 进 `adas_moon3d_v1`。 +ADAS cuboid 详见 [ADAS_MOON3D_PACK.md](./ADAS_MOON3D_PACK.md)：导出（支持 cuboid 3D / 2D 矩形 / YOLO HBB 回退）→ 可选 `cuboid_fit_3d` → `build_adas` 进 `adas_v1`。 **登记 meta**（`更多操作` 内）只写入 `batch.meta.yaml`，**不等于**已送标。 @@ -191,6 +191,7 @@ datasets/dms/inbox/dam/batch_0516/ |------|--------|--------|------| | **训练包**（默认） | `dms_v1` 等 | `dam` / `addw_face` / `前向·粗检测` / `前向·细分类` … | 该包下各任务的 train/val 快照；前向在 registry 为单任务双 mode，目录按 mode 拆开显示 | | **采集批次** | 任务（如 `dam`） | `0516 批次` 等 mode | 单波送标质量 | +| **前向** | `adas_v1` 等 pack | 批次列表 | ADAS 3D/2D 标注批次，含图片数、标注数、类别分布；支持 quaternion_json 和 YOLO HBB | | **车道线** | lane pack | — | Lane 列表统计 | 增量条（合并前）仍在磁盘 `packs///sources/<批次名>/`；合并用 `python HSAP/as.py build dms dam --pack dms_v1 --all-sources`。 diff --git a/platform/as_platform/audit/review.py b/platform/as_platform/audit/review.py index 3193174..71a1a69 100644 --- a/platform/as_platform/audit/review.py +++ b/platform/as_platform/audit/review.py @@ -68,7 +68,13 @@ def _class_names_for_campaign(camp) -> dict[int, str]: import yaml from as_platform.data.core import load_wf, proj_root - if not camp or camp.project != "dms": + if not camp: + return {} + if camp.project == "adas": + from as_platform.labeling.class_map import load_adas_class_names + names = load_adas_class_names() + return {i: n for i, n in enumerate(names)} + if camp.project != "dms": return {} wf = load_wf() root = proj_root(wf, "dms") @@ -114,21 +120,42 @@ def _parse_ls_annotations(path: Path, class_names: dict[int, str]) -> list[dict[ return [] out: list[dict[str, Any]] = [] for item in data.get("result") or []: - if item.get("type") not in ("rectanglelabels", "rectangle"): - continue - val = item.get("value") or {} - w_pct = float(val.get("width") or 0) - h_pct = float(val.get("height") or 0) - if w_pct <= 0 or h_pct <= 0: - continue - x_pct = float(val.get("x") or 0) - y_pct = float(val.get("y") or 0) - labels = val.get("rectanglelabels") or val.get("labels") or [] - label = labels[0] if labels else "unknown" - cid = _name_to_class_id(str(label), class_names) - cx = (x_pct + w_pct / 2) / 100.0 - cy = (y_pct + h_pct / 2) / 100.0 - out.append({"class_id": cid, "bbox": (cx, cy, w_pct / 100.0, h_pct / 100.0)}) + item_type = item.get("type") + if item_type == "cuboid": + # Cuboid: 8 vertices in pixel coords → compute 2D bbox + points = item.get("points") or [] + if len(points) < 16: + continue + label = item.get("label") or "unknown" + cid = _name_to_class_id(str(label), class_names) + # Compute min/max from 8 (x,y) pairs + xs = [points[i] for i in range(0, len(points), 2)] + ys = [points[i] for i in range(1, len(points), 2)] + min_x, max_x = min(xs), max(xs) + min_y, max_y = min(ys), max(ys) + if max_x <= min_x or max_y <= min_y: + continue + orig_w = float(item.get("original_width") or 1920) + orig_h = float(item.get("original_height") or 1080) + bw = (max_x - min_x) / orig_w + bh = (max_y - min_y) / orig_h + cx = (min_x + max_x) / 2.0 / orig_w + cy = (min_y + max_y) / 2.0 / orig_h + out.append({"class_id": cid, "bbox": (cx, cy, bw, bh)}) + elif item_type in ("rectanglelabels", "rectangle"): + val = item.get("value") or {} + w_pct = float(val.get("width") or 0) + h_pct = float(val.get("height") or 0) + if w_pct <= 0 or h_pct <= 0: + continue + x_pct = float(val.get("x") or 0) + y_pct = float(val.get("y") or 0) + labels = val.get("rectanglelabels") or val.get("labels") or [] + label = labels[0] if labels else "unknown" + cid = _name_to_class_id(str(label), class_names) + cx = (x_pct + w_pct / 2) / 100.0 + cy = (y_pct + h_pct / 2) / 100.0 + out.append({"class_id": cid, "bbox": (cx, cy, w_pct / 100.0, h_pct / 100.0)}) return out diff --git a/platform/as_platform/data/catalog_cache.py b/platform/as_platform/data/catalog_cache.py index d2d7c9e..3a5b019 100644 --- a/platform/as_platform/data/catalog_cache.py +++ b/platform/as_platform/data/catalog_cache.py @@ -83,7 +83,7 @@ def build_catalog_signature(wf: dict, proj_root_fn) -> dict[str, Any]: files.append({"path": str(p), "missing": True}) dirs: list[dict[str, Any]] = [] - for pname in ("dms", "lane"): + for pname in ("dms", "lane", "adas"): root = proj_root_fn(wf, pname) dirs.append(_dir_fingerprint(root, scan_children=False)) dirs.append(_dir_fingerprint(root / "inbox")) diff --git a/platform/as_platform/data/core.py b/platform/as_platform/data/core.py index af6e193..ef670e9 100644 --- a/platform/as_platform/data/core.py +++ b/platform/as_platform/data/core.py @@ -11,7 +11,7 @@ from typing import Any import yaml from as_platform.config import WORKSPACE, WORKSPACE_ROOT, LANE_DATA_VIZ_ENABLED -from as_platform.data.batch import META_FILENAME, dms_has_images, enrich_batch, write_meta +from as_platform.data.batch import META_FILENAME, dms_has_images, enrich_batch, read_meta, write_meta from as_platform.data.catalog_cache import ( build_catalog_signature, get_cached_catalog, @@ -811,7 +811,7 @@ def _normalize_catalog_dms(dms: dict[str, Any], reg: dict) -> dict[str, Any]: def _build_catalog(wf: dict, *, prefer_reports: bool = True) -> tuple[dict[str, Any], str]: - out: dict[str, Any] = {"workspace": str(WORKSPACE), "dms": {}, "lane": {}} + out: dict[str, Any] = {"workspace": str(WORKSPACE), "dms": {}, "lane": {}, "adas": {}} build_source = "scan" reports = load_dms_reports() @@ -966,6 +966,71 @@ def _build_catalog(wf: dict, *, prefer_reports: bool = True) -> tuple[dict[str, "add_template": "python as.py add lane --src --engineer --date YYYYMMDD", "quality": lane_quality, } + + # ── ADAS catalog ── + adas_root = proj_root(wf, "adas") + out["adas"] = {} + adas_packs_dir = adas_root / "packs" + if adas_packs_dir.is_dir(): + for pack_dir in sorted(adas_packs_dir.iterdir()): + if pack_dir.name.startswith(".") or not pack_dir.is_dir(): + continue + pack_name = pack_dir.name + sources_dir = pack_dir / "sources" + pack_entry: dict[str, Any] = { + "name": pack_name, + "path": str(pack_dir.relative_to(adas_root)), + "enabled": pack_name in wf["projects"]["adas"].get("active_packs", []), + "batches": [], + "total_images": 0, + "total_labels": 0, + "class_counts": {}, + } + if sources_dir.is_dir(): + for batch_dir in sorted(sources_dir.iterdir()): + if batch_dir.name.startswith(".") or not batch_dir.is_dir(): + continue + batch_name = batch_dir.name + meta = read_meta(batch_dir) or {} + stage = meta.get("stage", "unknown") + task = meta.get("task", "") + counts = meta.get("counts", {}) + img_count = counts.get("images", 0) + # Count quaternion_json detections + qdir = batch_dir / "labels" / "quaternion_json" + label_count = 0 + class_counts: dict[str, int] = {} + if qdir.is_dir(): + for qf in sorted(qdir.glob("*.json")): + try: + qdata = json.loads(qf.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + continue + for det in qdata.get("detections") or []: + label_count += 1 + cls = str(det.get("class_name") or det.get("class_id", "unknown")) + class_counts[cls] = class_counts.get(cls, 0) + 1 + # Count images from directory if meta is missing + if img_count == 0: + imgs_dir = batch_dir / "images" + if imgs_dir.is_dir(): + img_count = sum(1 for f in imgs_dir.rglob("*") if f.suffix.lower() in IMG_EXTS) + batch_entry = { + "batch": batch_name, + "task": task, + "stage": stage, + "images": img_count, + "labels": label_count, + "class_counts": class_counts, + "ingested_at": meta.get("ingested_at"), + } + pack_entry["batches"].append(batch_entry) + pack_entry["total_images"] += img_count + pack_entry["total_labels"] += label_count + for cls, n in class_counts.items(): + pack_entry["class_counts"][cls] = pack_entry["class_counts"].get(cls, 0) + n + out["adas"][pack_name] = pack_entry + return out, build_source @@ -996,7 +1061,9 @@ def get_catalog( result = {"task": task_or_pack, **(full_catalog.get("dms", {}).get(task_or_pack, {}))} elif project == "lane" and task_or_pack: result = {"pack": task_or_pack, **(full_catalog.get("lane", {}).get(task_or_pack, {}))} - elif project in ("dms", "lane"): + elif project == "adas" and task_or_pack: + result = {"pack": task_or_pack, **(full_catalog.get("adas", {}).get(task_or_pack, {}))} + elif project in ("dms", "lane", "adas"): result = {"workspace": full_catalog.get("workspace", str(WORKSPACE)), project: full_catalog.get(project, {})} else: result = dict(full_catalog) diff --git a/platform/as_platform/data/promote/adas_cuboid.py b/platform/as_platform/data/promote/adas_cuboid.py index 30dc7b5..a865e33 100644 --- a/platform/as_platform/data/promote/adas_cuboid.py +++ b/platform/as_platform/data/promote/adas_cuboid.py @@ -86,14 +86,20 @@ class AdasCuboidPromoteAdapter(PackPromoteAdapter): def promote(self, ctx: PromoteContext) -> PromoteResult: warnings = list(ctx.extra.get("validate_warnings") or []) qdir = ctx.batch_dir / "labels" / "quaternion_json" - if not qdir.is_dir() or not any(qdir.glob("*.json")): + has_quat = qdir.is_dir() and any(qdir.glob("*.json")) + + # YOLO HBB 回退 + labels_dir = ctx.batch_dir / "labels" + has_yolo = (labels_dir / "yolo-hbb").is_dir() and any((labels_dir / "yolo-hbb").glob("*.txt")) + + if not has_quat and not has_yolo: return PromoteResult( ok=False, project=ctx.project, task=ctx.task, batch=ctx.batch, pack=ctx.pack, - warnings=["missing quaternion_json export"], + warnings=["missing quaternion_json or YOLO HBB export"], ) pack_dir = ctx.project_root / "packs" / ctx.pack @@ -119,7 +125,9 @@ class AdasCuboidPromoteAdapter(PackPromoteAdapter): if src_sub.is_dir(): copied += _sync_tree(src_sub, dest / sub) - normalized = _normalize_quaternion_json(dest) + normalized = _normalize_quaternion_json(dest) if has_quat else 0 + if has_yolo and not has_quat: + warnings.append("using YOLO HBB labels (no quaternion_json)") meta = read_meta(ctx.batch_dir) or {} meta.update({ @@ -130,6 +138,7 @@ class AdasCuboidPromoteAdapter(PackPromoteAdapter): "pack": ctx.pack, "ingested_at": datetime.now(timezone.utc).isoformat(), "pipeline_version": 2, + "label_format": "quaternion_json" if has_quat else "yolo_hbb", }) write_meta(dest, meta) write_meta(ctx.batch_dir, meta) diff --git a/platform/as_platform/data/promote/validate/adas_cuboid.py b/platform/as_platform/data/promote/validate/adas_cuboid.py index b5e1fba..b0b6379 100644 --- a/platform/as_platform/data/promote/validate/adas_cuboid.py +++ b/platform/as_platform/data/promote/validate/adas_cuboid.py @@ -1,4 +1,4 @@ -"""ADAS cuboid batch validation before promote.""" +"""ADAS batch validation before promote（支持 quaternion_json 和 YOLO HBB）。""" from __future__ import annotations import json @@ -7,20 +7,87 @@ from pathlib import Path from as_platform.labeling.class_map import load_adas_class_names +def _has_yolo_hbb_labels(batch_dir: Path) -> bool: + """检查是否有 YOLO HBB txt 标注文件。""" + yolo_dir = batch_dir / "labels" / "yolo-hbb" + return yolo_dir.is_dir() and any(yolo_dir.glob("*.txt")) + + +def _validate_yolo_hbb(batch_dir: Path, expected_names: list[str]) -> tuple[list[str], list[str], dict]: + """校验 YOLO HBB 格式标注。""" + errors: list[str] = [] + warnings: list[str] = [] + n_classes = len(expected_names) + + labels_dir = batch_dir / "labels" / "yolo-hbb" + txt_files = sorted(labels_dir.glob("*.txt")) if labels_dir.is_dir() else [] + + total_boxes = 0 + files_with_boxes = 0 + for p in txt_files: + try: + lines = p.read_text(encoding="utf-8").strip().splitlines() + except OSError as e: + errors.append(f"{p.name}: read error ({e})") + continue + if not lines: + continue + has_valid = False + for line in lines: + line = line.strip() + if not line: + continue + parts = line.split() + if len(parts) < 5: + warnings.append(f"{p.name}: invalid YOLO line '{line[:40]}...'") + continue + try: + cid = int(parts[0]) + if cid < 0 or cid >= n_classes: + errors.append(f"{p.name}: class_id {cid} out of range (0-{n_classes - 1})") + continue + # 验证 bbox 值在 [0,1] + for v in parts[1:5]: + fv = float(v) + if fv < 0 or fv > 1: + warnings.append(f"{p.name}: bbox value {fv} out of [0,1]") + except ValueError: + warnings.append(f"{p.name}: non-numeric value in '{line[:40]}...'") + continue + total_boxes += 1 + has_valid = True + if has_valid: + files_with_boxes += 1 + + if files_with_boxes == 0: + errors.append("no valid YOLO HBB label files") + + stats = { + "format": "yolo_hbb", + "label_files": len(txt_files), + "files_with_boxes": files_with_boxes, + "total_boxes": total_boxes, + } + return errors, warnings, stats + + def validate_adas_cuboid_batch( batch_dir: Path, *, allow_partial_3d: bool = False, min_fit_ratio: float = 0.8, ) -> tuple[list[str], list[str], dict]: - """Return (errors, warnings, stats).""" + """Return (errors, warnings, stats). 优先 quaternion_json，其次 YOLO HBB。""" errors: list[str] = [] warnings: list[str] = [] qdir = batch_dir / "labels" / "quaternion_json" expected_names = load_adas_class_names() if not qdir.is_dir(): - errors.append(f"missing labels/quaternion_json under {batch_dir}") + # 回退到 YOLO HBB 格式 + if _has_yolo_hbb_labels(batch_dir): + return _validate_yolo_hbb(batch_dir, expected_names) + errors.append(f"missing labels/quaternion_json or YOLO HBB labels under {batch_dir}") return errors, warnings, {} files = sorted(qdir.glob("*.json")) diff --git a/platform/as_platform/deliveries/scan.py b/platform/as_platform/deliveries/scan.py index d4d86f3..7621769 100644 --- a/platform/as_platform/deliveries/scan.py +++ b/platform/as_platform/deliveries/scan.py @@ -34,7 +34,13 @@ def _scan_project_inbox(project: str, wf: dict | None = None) -> list[dict[str, with session_scope() as db: deliveries = { - (r.project, r.task or "", r.mode or "", r.batch_name): r + (r.project, r.task or "", r.mode or "", r.batch_name): { + "id": r.id, + "status": r.status, + "collection_start": r.collection_start, + "collection_end": r.collection_end, + "created_at": r.created_at, + } for r in db.query(BatchDelivery).filter(BatchDelivery.project == project).all() } indexed = { @@ -78,13 +84,13 @@ def _scan_project_inbox(project: str, wf: dict | None = None) -> list[dict[str, "has_labels": has_labels, "stage_hint": stage_hint, "source_type": "inbox_scan", - "delivery_id": delivery.id if delivery else None, - "delivery_status": delivery.status if delivery else None, + "delivery_id": delivery["id"] if delivery else None, + "delivery_status": delivery["status"] if delivery else None, "in_ledger": delivery is not None, "in_workbench": in_index, - "collection_start": delivery.collection_start if delivery else _dir_mtime_iso(batch_dir), - "collection_end": delivery.collection_end if delivery else None, - "created_at": delivery.created_at.isoformat() if delivery and delivery.created_at else None, + "collection_start": delivery["collection_start"] if delivery else _dir_mtime_iso(batch_dir), + "collection_end": delivery["collection_end"] if delivery else None, + "created_at": delivery["created_at"].isoformat() if delivery and delivery["created_at"] else None, "needs_ledger": delivery is None, "needs_workbench": not in_index, }) diff --git a/platform/as_platform/jobs/runner.py b/platform/as_platform/jobs/runner.py index ea7775e..ef31ca9 100644 --- a/platform/as_platform/jobs/runner.py +++ b/platform/as_platform/jobs/runner.py @@ -277,6 +277,24 @@ def execute_action(action: str, params: dict[str, Any]) -> dict[str, Any]: f"skipped_empty={conv.get('skipped_empty')} missing_ann={conv.get('missing_ann')}" ) return {"ok": True, "stdout": json.dumps(conv, ensure_ascii=False), "stderr": "", "export_convert": conv} + if row.get("project") == "adas" and export == "yolo": + scripts_dir = WORKSPACE / "datasets" / "adas" / "scripts" + if str(scripts_dir) not in sys.path: + sys.path.insert(0, str(scripts_dir)) + from export_ls_to_yolo import export_batch as export_adas_yolo + + with session_scope() as db: + camp = db.get(LabelingCampaign, campaign_id) + if not camp: + raise ValueError("campaign not found") + batch_dir = resolve_campaign_batch_dir(camp) + conv = export_adas_yolo(batch_dir, task=task, out_subdir="labels/yolo-hbb") + if conv.get("written", 0) == 0: + raise ValueError( + "export_adas_yolo: 无有效 YOLO HBB 标注可导出 (written=0); " + f"skipped_empty={conv.get('skipped_empty')} missing_ann={conv.get('missing_ann')}" + ) + return {"ok": True, "stdout": json.dumps(conv, ensure_ascii=False), "stderr": "", "export_convert": conv} if row.get("project") == "adas" and export == "cvat_cuboid": from as_platform.labeling.export_cuboid_batch import export_batch as export_cuboid_batch diff --git a/platform/as_platform/labeling/batch_stage.py b/platform/as_platform/labeling/batch_stage.py index d3cb30a..fce8869 100644 --- a/platform/as_platform/labeling/batch_stage.py +++ b/platform/as_platform/labeling/batch_stage.py @@ -119,8 +119,8 @@ def on_labeling_export_job_succeeded(job: dict) -> None: except Exception: return project = camp.project or "dms" - if _batch_has_export_labels(project, batch_dir): - _advance_campaign_stage(str(cid), "returned") + # job 已成功，无论是否有 CLI 导出产物，均进入 returned 阶段 + _advance_campaign_stage(str(cid), "returned") if project == "adas" and _batch_has_calib(batch_dir): from as_platform.jobs.queue import enqueue_job diff --git a/platform/as_platform/labeling/export_cuboid_batch.py b/platform/as_platform/labeling/export_cuboid_batch.py index 767bb1d..9bfd185 100644 --- a/platform/as_platform/labeling/export_cuboid_batch.py +++ b/platform/as_platform/labeling/export_cuboid_batch.py @@ -102,8 +102,45 @@ def _resolve_image_for_ann(data: dict[str, Any], batch_dir: Path, task_id: str) return None +def _rect_item_to_detection(item: dict[str, Any], class_map: dict[str, int]) -> dict[str, Any] | None: + """2D rectanglelabels 条目 → quaternion_json detection（无 3D 信息，仅 bbox2d）。""" + value = item.get("value") or {} + w_pct = float(value.get("width") or 0) + h_pct = float(value.get("height") or 0) + if w_pct <= 0 or h_pct <= 0: + return None + labels = value.get("rectanglelabels") or [] + label = str(labels[0]) if labels else "unknown" + class_id = class_map.get(label) + if class_id is None: + for name, cid in class_map.items(): + if name.lower() == label.lower(): + class_id = cid + break + if class_id is None: + return None + + img_w = int(item.get("original_width") or 1920) + img_h = int(item.get("original_height") or 1080) + x_pct = float(value.get("x") or 0) + y_pct = float(value.get("y") or 0) + x1 = int(x_pct / 100.0 * img_w) + y1 = int(y_pct / 100.0 * img_h) + x2 = int((x_pct + w_pct) / 100.0 * img_w) + y2 = int((y_pct + h_pct) / 100.0 * img_h) + + return { + "class_id": class_id, + "class_name": label, + "score": 1.0, + "box2d_xyxy": [x1, y1, x2, y2], + "fit_ok": False, + "source": "2d_rect", + } + + def export_batch(batch_dir: Path) -> dict[str, Any]: - """导出 cuboid ls_annotations → quaternion_json。""" + """导出 ls_annotations（cuboid + 2D rectangle）→ quaternion_json。""" batch_dir = batch_dir.resolve() class_map = _load_cuboid_class_map() calib_path, K, calib_size = _find_calib(batch_dir) @@ -124,7 +161,9 @@ def export_batch(batch_dir: Path) -> dict[str, Any]: continue regions = _extract_result_regions(data) cuboids = [r for r in regions if r.get("type") == "cuboid"] - if not cuboids: + rectangles = [r for r in regions if r.get("type") == "rectanglelabels"] + + if not cuboids and not rectangles: skipped_empty += 1 continue @@ -134,16 +173,24 @@ def export_batch(batch_dir: Path) -> dict[str, Any]: continue detections: list[dict[str, Any]] = [] + # 处理 cuboid 标注 for item in cuboids: det = cuboid_item_to_detection(item, class_map, K=K) if det: detections.append(det) + # 处理 2D 矩形标注 → quaternion_json detection + for item in rectangles: + det = _rect_item_to_detection(item, class_map) + if det: + detections.append(det) if not detections: skipped_empty += 1 continue - img_w = int((cuboids[0].get("original_width") or (calib_size or [1920, 1080])[0])) - img_h = int((cuboids[0].get("original_height") or (calib_size or [1920, 1080])[1])) + # 获取图片尺寸：优先 cuboid，其次 rectangle + first_region = cuboids[0] if cuboids else rectangles[0] + img_w = int((first_region.get("original_width") or (calib_size or [1920, 1080])[0])) + img_h = int((first_region.get("original_height") or (calib_size or [1920, 1080])[1])) payload: dict[str, Any] = { "image": str(image_path), diff --git a/platform/as_platform/labeling/format_converter.py b/platform/as_platform/labeling/format_converter.py index 2be4328..2a07509 100644 --- a/platform/as_platform/labeling/format_converter.py +++ b/platform/as_platform/labeling/format_converter.py @@ -238,10 +238,14 @@ def cvat_shapes_to_export_regions( } if stype == "rectangle": - xtl = float(shape.get("xtl", 0)) - ytl = float(shape.get("ytl", 0)) - xbr = float(shape.get("xbr", 0)) - ybr = float(shape.get("ybr", 0)) + pts = shape.get("points") or [] + if len(pts) >= 4: + xtl, ytl, xbr, ybr = float(pts[0]), float(pts[1]), float(pts[2]), float(pts[3]) + else: + xtl = float(shape.get("xtl", 0)) + ytl = float(shape.get("ytl", 0)) + xbr = float(shape.get("xbr", 0)) + ybr = float(shape.get("ybr", 0)) regions.append({ **base, "type": "rectanglelabels", diff --git a/platform/as_platform/labeling/service.py b/platform/as_platform/labeling/service.py index 3b625cc..18f5513 100644 --- a/platform/as_platform/labeling/service.py +++ b/platform/as_platform/labeling/service.py @@ -186,9 +186,9 @@ def open_campaign( else: camp.status = "in_progress" camp.updated_at = now - if cvat_task_id and not camp.cvat_task_id: - camp.cvat_task_id = cvat_task_id - camp.cvat_job_url = cvat_job_url + # 每次开标都创建新 CVAT Task，始终更新 task_id + camp.cvat_task_id = cvat_task_id + camp.cvat_job_url = cvat_job_url if ann_types and not camp.annotation_types: camp.annotation_types = ann_types db.flush() @@ -531,6 +531,7 @@ def _cvat_upload_thread(cvat_task_id: int, image_paths: list, campaign_id: str | with session_scope() as db: camp = db.get(LabelingCampaign, campaign_id) if camp and camp.cvat_job_url is None: + camp.cvat_task_id = cvat_task_id camp.cvat_job_url = f"_UPLOAD_FAILED_: {e}" camp.status = "upload_failed" updated = True @@ -542,12 +543,13 @@ def _cvat_upload_thread(cvat_task_id: int, image_paths: list, campaign_id: str | try: with session_scope() as db: camp = db.get(LabelingCampaign, campaign_id) - # 上传成功后，尝试回填 job_url + # 上传成功后，回填 job_url 和 task_id if camp and camp.cvat_job_url is None: from as_platform.labeling.cvat_client import get_cvat_client cvat = get_cvat_client() task = cvat.get_task(cvat_task_id) if task.job_url: + camp.cvat_task_id = cvat_task_id camp.cvat_job_url = task.job_url camp.status = "in_progress" updated = True @@ -657,7 +659,11 @@ def _build_class_map(camp, reg: dict | None) -> dict[str, int]: def get_cvat_status(campaign_id: str) -> dict[str, Any]: - """查询 CVAT 侧 Task 状态，包含上传进度信息。""" + """查询 CVAT 侧 Task 状态，包含上传进度信息。 + + 自动恢复：当检测到 cvat_job_url 为 None 且 CVAT Task 无数据时， + 说明上传线程已丢失（如容器重启），自动重新触发上传。 + """ with session_scope() as db: camp = db.get(LabelingCampaign, campaign_id) if not camp: @@ -690,12 +696,31 @@ def get_cvat_status(campaign_id: str) -> dict[str, Any]: images = _iter_batch_images(batch_dir) image_count = len(images) except Exception: + images = [] pass + + # 自动恢复：cvat_job_url 为空且 CVAT Task 无 Job → 上传线程已丢失 + resolved_url = task.job_url or camp.cvat_job_url + if not resolved_url and image_count > 0: + # CVAT Task 存在但无数据，重新触发上传 + import threading + _needs_recovery = True + try: + r = cvat._session.get(f"{cvat._api}/tasks/{camp.cvat_task_id}/data/meta") + _needs_recovery = r.status_code >= 400 # 400 = 未上传数据 + except Exception: + pass + if _needs_recovery: + print(f"[CVAT] 检测到断线任务，自动恢复上传 task={camp.cvat_task_id} campaign={campaign_id}") + uploader = _cvat_upload_thread(camp.cvat_task_id, images, campaign_id=campaign_id) + threading.Thread(target=uploader, daemon=True).start() + camp.cvat_job_url = None # 重置，等待线程回填 + return { "cvat_available": True, "campaign_id": campaign_id, "cvat_task_id": camp.cvat_task_id, - "cvat_job_url": task.job_url or camp.cvat_job_url, + "cvat_job_url": resolved_url, "cvat_status": task.status, "image_count": image_count, } diff --git a/platform/web/src/app/hsap-api.ts b/platform/web/src/app/hsap-api.ts index 2e09f56..ddbe73a 100644 --- a/platform/web/src/app/hsap-api.ts +++ b/platform/web/src/app/hsap-api.ts @@ -396,18 +396,18 @@ export const hsapApi = { if (opts?.status) p.set("status", opts.status); if (opts?.offset != null) p.set("offset", String(opts.offset)); if (opts?.limit != null) p.set("limit", String(opts.limit)); - return fetchJson>>(`${API_BASE}/api/v1/approvals?${p}`); + return fetchJson>>(`${API_BASE}/api/v1/system/audit?${p}`); }, - getApproval: (id: string) => fetchJson>(`${API_BASE}/api/v1/approvals/${id}`), - getApprovalPreview: (id: string) => fetchJson>(`${API_BASE}/api/v1/approvals/${id}/preview`), + getApproval: (id: string) => fetchJson>(`${API_BASE}/api/v1/system/audit/${id}`), + getApprovalPreview: (id: string) => fetchJson>(`${API_BASE}/api/v1/system/audit/${id}/preview`), listApprovalImages: (id: string, offset = 0, limit = 60) => - fetchJson<{ total: number; items: { id: string }[] }>(`${API_BASE}/api/v1/approvals/${id}/images?offset=${offset}&limit=${limit}`), + fetchJson<{ total: number; items: { id: string }[] }>(`${API_BASE}/api/v1/system/audit/${id}/images?offset=${offset}&limit=${limit}`), fetchApprovalImageBlob: async (approvalId: string, imageId: string, thumb = true) => { const q = thumb ? "?thumb=true" : "?thumb=false"; - const res = await fetch(`${API_BASE}/api/v1/approvals/${approvalId}/images/${imageId}${q}`, { headers: authHeaders(), cache: "no-store" }); + const res = await fetch(`${API_BASE}/api/v1/system/audit/${approvalId}/images/${imageId}${q}`, { headers: authHeaders(), cache: "no-store" }); if (!res.ok) throw new Error(await res.text()); return URL.createObjectURL(await res.blob()); }, diff --git a/platform/web/src/lib/dmsCatalog.ts b/platform/web/src/lib/dmsCatalog.ts index 3373e4c..865d006 100644 --- a/platform/web/src/lib/dmsCatalog.ts +++ b/platform/web/src/lib/dmsCatalog.ts @@ -2,6 +2,7 @@ export type CatalogReport = Record & { dms?: Record; lane?: Record>; + adas?: Record; projects?: { dms?: { active_packs?: string[] }; lane?: { active_packs?: string[] } }; _cache?: Record; }; @@ -19,6 +20,28 @@ export type DmsPackRow = { total_boxes?: number; label_files?: number; sampled?: boolean; }; +// ── ADAS catalog types ── + +export type AdasPackEntry = { + name: string; + path: string; + enabled: boolean; + total_images: number; + total_labels: number; + class_counts: Record; + batches: AdasBatchEntry[]; +}; + +export type AdasBatchEntry = { + batch: string; + task: string; + stage: string; + images: number; + labels: number; + class_counts: Record; + ingested_at?: string; +}; + export type SplitCounts = { train: number; val: number; test: number }; export function aggregateSplitCounts(packs: DmsPackRow[]): SplitCounts { diff --git a/platform/web/src/modules/labeling/components/ExportBatchTable.tsx b/platform/web/src/modules/labeling/components/ExportBatchTable.tsx index a6cdaf1..52cd4fd 100644 --- a/platform/web/src/modules/labeling/components/ExportBatchTable.tsx +++ b/platform/web/src/modules/labeling/components/ExportBatchTable.tsx @@ -10,6 +10,7 @@ import type { LabelingBatchRow } from "@/lib/types"; type ExportBatchTableProps = { batches: LabelingBatchRow[]; importingId: string | null; + exportingId: string | null; buildingId: string | null; onExport: (campaignId: string) => void; onImportVendor: (campaignId: string) => void; @@ -21,6 +22,7 @@ const COLS = ["34%", "14%", "8%", "10%", "auto", "7.5rem"]; export const ExportBatchTable: React.FC = ({ batches, importingId, + exportingId, buildingId, onExport, onImportVendor, @@ -67,7 +69,7 @@ export const ExportBatchTable: React.FC = ({ {cid && isExport && (
-
- {/* DMS / Forward sub-views */} - {domain !== "lane" && ( + {/* DMS sub-views */} + {domain === "dms" && (

视图

{(["pack", "batch"] as const).map((v) => ( @@ -198,8 +199,8 @@ export const CatalogPage: React.FC = () => {
)} - {/* Pack selector */} - {domain !== "lane" && subView === "pack" && packTree.length > 0 && ( + {/* Pack selector — only for DMS */} + {domain === "dms" && subView === "pack" && packTree.length > 0 && (