feat: Unified Ingest SDK for DMS/ADAS promote, cuboid export and 3D fit

Replace subprocess build with promote_batch SDK, add ADAS cuboid export/fit/validate pipeline, stage normalization, and offline unit tests wired into smoke_labeling_api.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
2026-06-16 09:58:35 +08:00
parent bc653d53a1
commit 0b8ade048e
42 changed files with 2074 additions and 104 deletions

View File

@@ -37,6 +37,8 @@ from as_platform.labeling.service import (
open_campaign,
submit_campaign,
trigger_labeling_export,
get_batch_export_stats,
trigger_cuboid_fit,
)
from as_platform.labeling.vendor_import import import_vendor_zip, list_registry_profiles
@@ -315,6 +317,30 @@ def api_labeling_export(
raise HTTPException(404, "campaign not found") from None
@router.get("/api/v1/labeling/campaigns/{campaign_id}/export-stats")
def api_batch_export_stats(
campaign_id: str,
_user: Annotated[User, Depends(require_permission("read:pending"))],
) -> dict[str, Any]:
try:
return get_batch_export_stats(campaign_id)
except FileNotFoundError:
raise HTTPException(404, "campaign not found") from None
@router.post("/api/v1/labeling/campaigns/{campaign_id}/cuboid-fit")
def api_cuboid_fit(
campaign_id: str,
_user: Annotated[User, Depends(require_permission("read:pending"))],
) -> dict[str, Any]:
try:
return trigger_cuboid_fit(campaign_id)
except FileNotFoundError:
raise HTTPException(404, "campaign not found") from None
except ValueError as e:
raise HTTPException(400, str(e)) from e
@router.get("/api/v1/labeling/campaigns/{campaign_id}/export-jobs")
def api_campaign_export_jobs(
campaign_id: str,

View File

@@ -498,15 +498,23 @@ def api_submit(body: SubmitApprovalBody, user: Annotated[User, Depends(get_curre
@app.post("/api/v1/approvals/submit-build-batch")
def api_submit_build_batch(body: BuildFromBatchBody, user: Annotated[User, Depends(get_current_user)]) -> dict[str, Any]:
if not can_submit_action(user, "build_dms"):
action = "build_adas" if body.project == "adas" else "build_dms"
if not can_submit_action(user, action) and not can_submit_action(user, "build_dms"):
raise HTTPException(403, "无权提交 build")
params: dict[str, Any] = {"task": body.task, "pack": body.pack}
pack = body.pack
if body.project == "adas" and (not pack or pack == "dms_v2"):
pack = "adas_moon3d_v1"
params: dict[str, Any] = {
"project": body.project,
"task": body.task,
"pack": pack,
}
if body.location == "inbox":
params["batch"] = body.batch
else:
params["all_sources"] = True
return submit_approval(
"build_dms", params,
action, params,
submitted_by=user.name,
submitted_by_user_id=user.id,
note=body.note or f"入库 {body.batch}",

View File

@@ -92,6 +92,47 @@ def api_system_submit_approval(
raise HTTPException(400, str(e)) from e
class BuildFromBatchBody(BaseModel):
project: str = "dms"
task: str
batch: str
pack: str = "dms_v2"
location: str = "inbox"
note: str | None = None
@router.post("/audit/submit-build-batch")
def api_system_submit_build_batch(
body: BuildFromBatchBody,
user: Annotated[User, Depends(get_current_user)],
) -> dict[str, Any]:
action = "build_adas" if body.project == "adas" else "build_dms"
if not can_submit_action(user, action) and not can_submit_action(user, "build_dms"):
raise HTTPException(403, "无权提交 build")
pack = body.pack
if body.project == "adas" and (not pack or pack == "dms_v2"):
pack = "adas_moon3d_v1"
params: dict[str, Any] = {
"project": body.project,
"task": body.task,
"pack": pack,
}
if body.location == "inbox":
params["batch"] = body.batch
else:
params["all_sources"] = True
try:
return submit_approval(
action,
params,
submitted_by=user.name,
submitted_by_user_id=user.id,
note=body.note or f"入库 {body.batch}",
)
except ValueError as e:
raise HTTPException(400, str(e)) from e
@router.get("/audit/{record_id}")
def api_system_get_approval(
record_id: str,

View File

@@ -264,6 +264,35 @@ def resolve_approval_scope(action: str, params: dict[str, Any]) -> dict[str, Any
"batches": batches,
}
if action == "build_adas":
task = p.get("task") or "cuboid_7cls"
batch_name = p.get("batch")
root = proj_root(wf, "adas")
batches: list[dict[str, Any]] = []
if batch_name:
batches.append({"path": root / "inbox" / task / batch_name, "batch": batch_name, "location": "inbox"})
pack = p.get("pack") or "adas_moon3d_v1"
stats: dict[str, Any] = {}
if batch_name:
from as_platform.data.promote.validate.adas_cuboid import validate_adas_cuboid_batch
bpath = root / "inbox" / task / batch_name
if bpath.is_dir():
_err, _warn, stats = validate_adas_cuboid_batch(bpath, allow_partial_3d=True)
from as_platform.labeling.class_map import load_adas_class_names
names = load_adas_class_names()
class_names = {i: n for i, n in enumerate(names)}
return {
"project": "adas",
"task": task,
"pack": pack,
"scope_label": f"ADAS · {task} · {pack}" + (f" · {batch_name}" if batch_name else ""),
"class_names": class_names,
"batches": batches,
"export_stats": stats,
}
if action == "delivery_ingest":
data_path = (p.get("data_path") or "").strip()
if not data_path:

View File

@@ -11,7 +11,7 @@ from as_platform.config import LANE_DATA_VIZ_ENABLED
from as_platform.integrations.feishu_notify import send_chat_async
ACTIONS_REQUIRING_APPROVAL = {
"build_dms", "build_lane", "enable_pack", "disable_pack",
"build_dms", "build_adas", "build_lane", "enable_pack", "disable_pack",
"train_dms", "train_lane", "eval_dms", "promote_dms",
"pipeline_dms", "register_batch", "eval_lane", "visualize_dms", "visualize_lane",
"delivery_ingest",
@@ -31,6 +31,7 @@ REJECTION_CATEGORY_LABEL = {k: v for k, v in REJECTION_CATEGORIES.items()}
ACTION_LABELS = {
"build_dms": "DMS 入库 (build)",
"build_adas": "ADAS 入库 (build)",
"build_lane": "车道线合并列表 (build lane)",
"enable_pack": "启用训练数据包",
"disable_pack": "停用训练数据包",

View File

@@ -272,9 +272,10 @@ def _update_campaign_stage(db, campaign_id: str, new_stage: str) -> None:
from as_platform.labeling.batch_stage import update_campaign_batch_meta_stage
camp = db.get(LabelingCampaign, campaign_id)
if camp:
camp.status = new_stage
effective = "labeling_submitted" if new_stage == "review_approved" else new_stage
camp.status = effective
db.flush()
update_campaign_batch_meta_stage(camp, new_stage)
update_campaign_batch_meta_stage(camp, effective)
def review_progress(campaign_id: str) -> dict[str, int]:

View File

@@ -0,0 +1,11 @@
from as_platform.data.promote.base import PackPromoteAdapter, PromoteContext, PromoteResult
from as_platform.data.promote.registry import get_promote_adapter
from as_platform.data.promote.runner import promote_batch
__all__ = [
"PackPromoteAdapter",
"PromoteContext",
"PromoteResult",
"get_promote_adapter",
"promote_batch",
]

View File

@@ -0,0 +1,152 @@
"""ADAS cuboid MOON-3D pack promote adapter."""
from __future__ import annotations
import json
import os
import shutil
from datetime import datetime, timezone
from pathlib import Path
from as_platform.data.batch import read_meta, write_meta
from as_platform.data.promote.base import PackPromoteAdapter, PromoteContext, PromoteResult
from as_platform.data.promote.manifest import refresh_adas_lists
from as_platform.data.promote.validate.adas_cuboid import validate_adas_cuboid_batch
from as_platform.labeling.class_map import build_class_map, load_adas_class_names, normalize_detection_class
IMG_EXTS = {".jpg", ".jpeg", ".png", ".bmp", ".webp"}
def _link_or_copy(src: Path, dst: Path, *, copy: bool = False) -> None:
dst.parent.mkdir(parents=True, exist_ok=True)
if dst.exists() or dst.is_symlink():
dst.unlink()
if copy:
if src.is_dir():
shutil.copytree(src, dst, dirs_exist_ok=True)
else:
shutil.copy2(src, dst)
return
try:
os.link(src, dst)
except OSError:
if src.is_dir():
shutil.copytree(src, dst, dirs_exist_ok=True)
else:
shutil.copy2(src, dst)
def _sync_tree(src: Path, dst: Path, *, copy: bool = False) -> int:
count = 0
if not src.is_dir():
return 0
for p in sorted(src.rglob("*")):
if not p.is_file():
continue
rel = p.relative_to(src)
target = dst / rel
if not target.exists():
_link_or_copy(p, target, copy=copy)
count += 1
return count
def _normalize_quaternion_json(dest_batch: Path) -> int:
qdir = dest_batch / "labels" / "quaternion_json"
if not qdir.is_dir():
return 0
cmap = build_class_map(load_adas_class_names())
names = load_adas_class_names()
updated = 0
for p in qdir.glob("*.json"):
data = json.loads(p.read_text(encoding="utf-8"))
dets = []
for det in data.get("detections") or []:
dets.append(normalize_detection_class(det, cmap))
data["detections"] = dets
data["text_prompts"] = names
data["num_detections"] = len(dets)
p.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
updated += 1
return updated
class AdasCuboidPromoteAdapter(PackPromoteAdapter):
project = "adas"
def validate(self, ctx: PromoteContext) -> list[str]:
if ctx.skip_validate:
return []
errors, warnings, _stats = validate_adas_cuboid_batch(
ctx.batch_dir,
allow_partial_3d=ctx.allow_partial_3d,
)
ctx.extra.setdefault("validate_warnings", warnings)
return errors
def promote(self, ctx: PromoteContext) -> PromoteResult:
warnings = list(ctx.extra.get("validate_warnings") or [])
qdir = ctx.batch_dir / "labels" / "quaternion_json"
if not qdir.is_dir() or not any(qdir.glob("*.json")):
return PromoteResult(
ok=False,
project=ctx.project,
task=ctx.task,
batch=ctx.batch,
pack=ctx.pack,
warnings=["missing quaternion_json export"],
)
pack_dir = ctx.project_root / "packs" / ctx.pack
dest = pack_dir / "sources" / ctx.batch
if ctx.dry_run:
return PromoteResult(
ok=True,
project=ctx.project,
task=ctx.task,
batch=ctx.batch,
pack=ctx.pack,
dest_path=str(dest),
detail={"dry_run": True},
)
if dest.exists():
shutil.rmtree(dest)
dest.mkdir(parents=True, exist_ok=True)
copied = 0
for sub in ("images", "calib", "labels"):
src_sub = ctx.batch_dir / sub
if src_sub.is_dir():
copied += _sync_tree(src_sub, dest / sub)
normalized = _normalize_quaternion_json(dest)
meta = read_meta(ctx.batch_dir) or {}
meta.update({
"stage": "ingested",
"project": ctx.project,
"task": ctx.task,
"batch": ctx.batch,
"pack": ctx.pack,
"ingested_at": datetime.now(timezone.utc).isoformat(),
"pipeline_version": 2,
})
write_meta(dest, meta)
write_meta(ctx.batch_dir, meta)
manifest = refresh_adas_lists(pack=ctx.pack)
img_count = sum(1 for _ in (dest / "images").rglob("*") if _.suffix.lower() in IMG_EXTS) if (dest / "images").is_dir() else 0
return PromoteResult(
ok=True,
project=ctx.project,
task=ctx.task,
batch=ctx.batch,
pack=ctx.pack,
dest_path=str(dest),
images=img_count,
labels=normalized,
manifest_paths=[manifest.get("train_list", ""), manifest.get("val_list", "")],
warnings=warnings,
detail={"copied_files": copied, "normalized_json": normalized, **manifest},
)

View File

@@ -0,0 +1,56 @@
"""Pack promote adapter base types."""
from __future__ import annotations
from abc import ABC, abstractmethod
from dataclasses import asdict, dataclass, field
from pathlib import Path
from typing import Any
@dataclass
class PromoteContext:
project: str
task: str
batch: str
pack: str
batch_dir: Path
project_root: Path
dry_run: bool = False
skip_validate: bool = False
allow_partial_3d: bool = False
refresh: bool = True
extra: dict[str, Any] = field(default_factory=dict)
@dataclass
class PromoteResult:
ok: bool
project: str
task: str
batch: str
pack: str
dest_path: str = ""
images: int = 0
labels: int = 0
manifest_paths: list[str] = field(default_factory=list)
warnings: list[str] = field(default_factory=list)
stage: str = "ingested"
detail: dict[str, Any] = field(default_factory=dict)
def to_dict(self) -> dict[str, Any]:
out = asdict(self)
out["ok"] = self.ok
return out
class PackPromoteAdapter(ABC):
project: str = ""
@abstractmethod
def promote(self, ctx: PromoteContext) -> PromoteResult:
raise NotImplementedError
@abstractmethod
def validate(self, ctx: PromoteContext) -> list[str]:
"""Return list of error messages; empty means pass."""
raise NotImplementedError

View File

@@ -0,0 +1,62 @@
"""DMS YOLO pack promote adapter."""
from __future__ import annotations
import sys
from pathlib import Path
from as_platform.data.promote.base import PackPromoteAdapter, PromoteContext, PromoteResult
from as_platform.data.promote.manifest import refresh_dms_yaml
from as_platform.data.promote.validate.dms_yolo import validate_dms_task
_DMS_SCRIPTS = Path(__file__).resolve().parents[4] / "datasets" / "dms" / "scripts"
if str(_DMS_SCRIPTS) not in sys.path:
sys.path.insert(0, str(_DMS_SCRIPTS))
class DmsYoloPromoteAdapter(PackPromoteAdapter):
project = "dms"
def validate(self, ctx: PromoteContext) -> list[str]:
if ctx.skip_validate:
return []
return validate_dms_task(ctx.task)
def promote(self, ctx: PromoteContext) -> PromoteResult:
from ingest_incremental import promote_inbox_batch
if not ctx.batch_dir.is_dir():
return PromoteResult(
ok=False,
project=ctx.project,
task=ctx.task,
batch=ctx.batch,
pack=ctx.pack,
warnings=[f"batch_dir missing: {ctx.batch_dir}"],
)
pack_dir = ctx.project_root / "packs" / ctx.pack
pack_dir.mkdir(parents=True, exist_ok=True)
detail = promote_inbox_batch(
root=ctx.project_root,
task=ctx.task,
pack=ctx.pack,
src=ctx.batch_dir,
mode=ctx.extra.get("mode"),
dry_run=ctx.dry_run,
refresh=ctx.refresh and not ctx.dry_run,
)
if ctx.refresh and not ctx.dry_run and not ctx.skip_validate:
refresh_dms_yaml(task=ctx.task)
added = int(detail.get("added") or 0)
return PromoteResult(
ok=True,
project=ctx.project,
task=ctx.task,
batch=ctx.batch,
pack=ctx.pack,
dest_path=str(ctx.project_root / "packs" / ctx.pack),
labels=added,
detail=detail,
)

View File

@@ -0,0 +1,93 @@
"""Refresh ADAS / DMS pack manifests after promote."""
from __future__ import annotations
import json
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
import yaml
from as_platform.data.core import load_wf, proj_root
def _collect_adas_stems(sources_root: Path) -> list[str]:
stems: list[str] = []
if not sources_root.is_dir():
return stems
for batch_dir in sorted(sources_root.iterdir()):
if not batch_dir.is_dir() or batch_dir.name.startswith("."):
continue
qdir = batch_dir / "labels" / "quaternion_json"
if qdir.is_dir():
for p in sorted(qdir.glob("*.json")):
stems.append(p.stem)
else:
img_root = batch_dir / "images"
if img_root.is_dir():
for p in sorted(img_root.rglob("*")):
if p.is_file() and p.suffix.lower() in {".jpg", ".jpeg", ".png"}:
stems.append(p.stem)
return sorted(set(stems))
def refresh_adas_lists(wf: dict | None = None, *, pack: str = "adas_moon3d_v1") -> dict[str, Any]:
wf = wf or load_wf()
root = proj_root(wf, "adas")
pack_dir = root / "packs" / pack
sources = pack_dir / "sources"
lists_dir = pack_dir / "lists"
lists_dir.mkdir(parents=True, exist_ok=True)
stems = _collect_adas_stems(sources)
val_ratio = 0.1
reg_path = root / wf["projects"]["adas"]["registry"]
if reg_path.is_file():
reg = yaml.safe_load(reg_path.read_text(encoding="utf-8")) or {}
val_ratio = float((reg.get("split") or {}).get("val_ratio", 0.1))
n_val = max(0, int(len(stems) * val_ratio)) if len(stems) > 1 else 0
val_stems = stems[:n_val]
train_stems = stems[n_val:]
train_path = lists_dir / "train_stems.txt"
val_path = lists_dir / "val_stems.txt"
train_path.write_text("\n".join(train_stems) + ("\n" if train_stems else ""), encoding="utf-8")
val_path.write_text("\n".join(val_stems) + ("\n" if val_stems else ""), encoding="utf-8")
manifest_dir = pack_dir / "manifests"
manifest_dir.mkdir(parents=True, exist_ok=True)
index_path = manifest_dir / "pack_index.yaml"
batches = []
if sources.is_dir():
for d in sorted(sources.iterdir()):
if d.is_dir() and not d.name.startswith("."):
batches.append({"batch": d.name, "path": str(d)})
index = {
"pack": pack,
"updated_at": datetime.now(timezone.utc).isoformat(),
"batches": batches,
"train_stems": len(train_stems),
"val_stems": len(val_stems),
}
index_path.write_text(yaml.dump(index, allow_unicode=True, sort_keys=False), encoding="utf-8")
return {
"train_list": str(train_path),
"val_list": str(val_path),
"pack_index": str(index_path),
"train_count": len(train_stems),
"val_count": len(val_stems),
}
def refresh_dms_yaml(wf: dict | None = None, task: str | None = None) -> None:
wf = wf or load_wf()
root = proj_root(wf, "dms")
import subprocess
import sys
cmd = [sys.executable, str(root / "scripts" / "refresh_yaml.py")]
if task:
cmd.extend(["--task", task])
subprocess.check_call(cmd, cwd=str(root))

View File

@@ -0,0 +1,18 @@
"""Pack promote adapter registry."""
from __future__ import annotations
from as_platform.data.promote.adas_cuboid import AdasCuboidPromoteAdapter
from as_platform.data.promote.base import PackPromoteAdapter
from as_platform.data.promote.dms_yolo import DmsYoloPromoteAdapter
ADAPTERS: tuple[PackPromoteAdapter, ...] = (
DmsYoloPromoteAdapter(),
AdasCuboidPromoteAdapter(),
)
def get_promote_adapter(project: str) -> PackPromoteAdapter:
for adapter in ADAPTERS:
if adapter.project == project:
return adapter
raise ValueError(f"no promote adapter for project={project}")

View File

@@ -0,0 +1,126 @@
"""Unified pack promote entrypoint."""
from __future__ import annotations
from pathlib import Path
from typing import Any
from as_platform.data.batch import read_meta, write_meta
from as_platform.data.catalog_cache import invalidate_catalog_cache
from as_platform.data.core import load_wf, proj_root
from as_platform.data.promote.base import PromoteContext, PromoteResult
from as_platform.data.promote.registry import get_promote_adapter
from as_platform.db.engine import session_scope
from as_platform.db.models import LabelingCampaign
from as_platform.jobs.runner import _auto_snapshot
from as_platform.labeling.annotate import resolve_campaign_batch_dir
def _resolve_batch_dir(
project: str,
task: str,
batch: str,
*,
location: str = "inbox",
) -> Path:
wf = load_wf()
root = proj_root(wf, project)
if location == "inbox":
if project == "adas":
return (root / "inbox" / task / batch).resolve()
return (root / "inbox" / task / batch).resolve()
raise ValueError(f"unsupported location: {location}")
def _update_campaign_ingested(project: str, task: str, batch: str) -> None:
try:
with session_scope() as db:
camp = (
db.query(LabelingCampaign)
.filter(
LabelingCampaign.project == project,
LabelingCampaign.task == task,
LabelingCampaign.batch == batch,
)
.order_by(LabelingCampaign.created_at.desc())
.first()
)
if camp:
camp.status = "ingested"
db.flush()
try:
batch_dir = resolve_campaign_batch_dir(camp)
meta = read_meta(batch_dir) or {}
meta["stage"] = "ingested"
meta["pipeline_version"] = 2
write_meta(batch_dir, meta)
except Exception:
pass
except Exception:
pass
def promote_batch(
project: str,
*,
task: str,
batch: str | None = None,
pack: str | None = None,
batch_dir: Path | str | None = None,
dry_run: bool = False,
skip_validate: bool = False,
allow_partial_3d: bool = False,
refresh: bool = True,
all_sources: bool = False,
extra: dict[str, Any] | None = None,
) -> dict[str, Any]:
"""Promote inbox batch into training pack (SDK entry)."""
wf = load_wf()
pcfg = wf["projects"][project]
pack_name = pack or pcfg.get("base_pack")
if not pack_name:
raise ValueError(f"project {project} missing pack")
if not task:
raise ValueError("task required")
if all_sources:
raise ValueError("all_sources promote not yet in SDK; use CLI ingest_incremental")
if not batch:
raise ValueError("batch required")
root = proj_root(wf, project)
bdir = Path(batch_dir).resolve() if batch_dir else _resolve_batch_dir(project, task, batch)
if not bdir.is_dir():
raise ValueError(f"batch_dir not found: {bdir}")
adapter = get_promote_adapter(project)
ctx = PromoteContext(
project=project,
task=task,
batch=batch,
pack=pack_name,
batch_dir=bdir,
project_root=root,
dry_run=dry_run,
skip_validate=skip_validate,
allow_partial_3d=allow_partial_3d,
refresh=refresh,
extra=extra or {},
)
val_errors = adapter.validate(ctx)
if val_errors:
raise ValueError("; ".join(val_errors))
result: PromoteResult = adapter.promote(ctx)
if not result.ok:
raise ValueError(result.warnings[0] if result.warnings else "promote failed")
if not dry_run:
_update_campaign_ingested(project, task, batch)
invalidate_catalog_cache()
if project == "dms":
_auto_snapshot("dms", task=task)
out = result.to_dict()
out["stdout"] = __import__("json").dumps(out, ensure_ascii=False)
out["stderr"] = ""
return out

View File

@@ -0,0 +1,81 @@
"""ADAS cuboid batch validation before promote."""
from __future__ import annotations
import json
from pathlib import Path
from as_platform.labeling.class_map import load_adas_class_names
def validate_adas_cuboid_batch(
batch_dir: Path,
*,
allow_partial_3d: bool = False,
min_fit_ratio: float = 0.8,
) -> tuple[list[str], list[str], dict]:
"""Return (errors, warnings, stats)."""
errors: list[str] = []
warnings: list[str] = []
qdir = batch_dir / "labels" / "quaternion_json"
expected_names = load_adas_class_names()
if not qdir.is_dir():
errors.append(f"missing labels/quaternion_json under {batch_dir}")
return errors, warnings, {}
files = sorted(qdir.glob("*.json"))
if not files:
errors.append("no quaternion_json files")
return errors, warnings, {}
total_dets = 0
fit_ok = 0
has_k = 0
files_with_dets = 0
for p in files:
try:
data = json.loads(p.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as e:
errors.append(f"{p.name}: invalid json ({e})")
continue
dets = data.get("detections") or []
if not dets:
warnings.append(f"{p.name}: empty detections (skipped)")
continue
files_with_dets += 1
if data.get("K"):
has_k += 1
prompts = data.get("text_prompts") or []
if prompts and list(prompts) != expected_names:
warnings.append(f"{p.name}: text_prompts order differs from registry")
for det in dets:
total_dets += 1
cid = det.get("class_id")
if cid is None or int(cid) < 0 or int(cid) >= len(expected_names):
errors.append(f"{p.name}: invalid class_id {cid}")
if det.get("fit_ok"):
fit_ok += 1
stats = {
"quaternion_files": len(files),
"files_with_detections": files_with_dets,
"detections": total_dets,
"fit_ok_ratio": fit_ok / max(total_dets, 1),
"has_k_ratio": has_k / max(files_with_dets, 1),
}
if files_with_dets == 0:
errors.append("no quaternion json with detections")
calib_dir = batch_dir / "calib"
if calib_dir.is_dir() and list(calib_dir.glob("*.yaml")):
if files_with_dets > 0 and has_k < files_with_dets:
errors.append(f"calib present but only {has_k}/{files_with_dets} annotated json have K")
if not allow_partial_3d and total_dets > 0:
ratio = fit_ok / total_dets
if ratio < min_fit_ratio:
errors.append(
f"fit_ok ratio {ratio:.2f} < {min_fit_ratio} (use allow_partial_3d for pilot)"
)
return errors, warnings, stats

View File

@@ -0,0 +1,18 @@
"""DMS YOLO batch validation wrapper."""
from __future__ import annotations
import subprocess
import sys
from pathlib import Path
from as_platform.config import WORKSPACE
def validate_dms_task(task: str | None) -> list[str]:
cmd = [sys.executable, str(WORKSPACE / "scripts" / "validate_dms_tasks.py")]
if task:
cmd.extend(["--task", task])
proc = subprocess.run(cmd, capture_output=True, text=True)
if proc.returncode != 0:
return [proc.stderr or proc.stdout or f"validate_dms_tasks failed exit {proc.returncode}"]
return []

View File

@@ -140,6 +140,10 @@ def _run_job(job_id: str) -> None:
from as_platform.labeling.batch_stage import on_labeling_export_job_succeeded
on_labeling_export_job_succeeded(job)
elif job.get("action") in ("build_dms", "build_adas", "build_lane"):
from as_platform.labeling.batch_stage import on_build_job_succeeded
on_build_job_succeeded(job)
except Exception as e:
_patch(job_id, status="failed", finished_at=_now(), result={"ok": False, "error": str(e)})
publish("job.failed", {"job_id": job_id, "error": str(e)})

View File

@@ -4,6 +4,7 @@ from __future__ import annotations
import json
import subprocess
import sys
from pathlib import Path
from typing import Any
from as_platform.config import WORKSPACE, PLATFORM_DIR, LANE_DATA_VIZ_ENABLED
@@ -73,24 +74,52 @@ def execute_action(action: str, params: dict[str, Any]) -> dict[str, Any]:
return _run_ml(["train", "lane"], timeout=86400)
if action == "build_dms":
argv = ["build", "dms", p["task"]]
if p.get("pack"):
argv.extend(["--pack", str(p["pack"])])
if p.get("batch"):
argv.extend(["--batch", str(p["batch"])])
if p.get("all_sources"):
argv.append("--all-sources")
if p.get("dry_run"):
argv.append("--dry-run")
if p.get("skip_validate"):
argv.append("--skip-validate")
if p.get("no_refresh"):
argv.append("--no-refresh")
result = _run_ml(argv)
# 自动创建数据集快照
_auto_snapshot("dms", task=p.get("task", ""))
from as_platform.data.promote.runner import promote_batch
result = promote_batch(
"dms",
task=p["task"],
batch=p.get("batch"),
pack=p.get("pack"),
dry_run=bool(p.get("dry_run")),
skip_validate=bool(p.get("skip_validate")),
refresh=not p.get("no_refresh"),
)
return result
if action == "build_adas":
from as_platform.data.promote.runner import promote_batch
return promote_batch(
"adas",
task=p.get("task", "cuboid_7cls"),
batch=p.get("batch"),
pack=p.get("pack", "adas_moon3d_v1"),
dry_run=bool(p.get("dry_run")),
skip_validate=bool(p.get("skip_validate")),
allow_partial_3d=bool(p.get("allow_partial_3d", True)),
)
if action == "cuboid_fit_3d":
from as_platform.db.engine import session_scope
from as_platform.db.models import LabelingCampaign
from as_platform.labeling.annotate import resolve_campaign_batch_dir
from as_platform.labeling.fit_cuboid_batch import fit_batch
campaign_id = p.get("campaign_id", "")
batch_dir = None
if campaign_id:
with session_scope() as db:
camp = db.get(LabelingCampaign, campaign_id)
if camp:
batch_dir = resolve_campaign_batch_dir(camp)
if batch_dir is None and p.get("batch_dir"):
batch_dir = Path(p["batch_dir"])
if batch_dir is None:
raise ValueError("cuboid_fit_3d 需要 campaign_id 或 batch_dir")
conv = fit_batch(batch_dir)
return {"ok": True, "stdout": json.dumps(conv, ensure_ascii=False), "stderr": "", "fit_convert": conv}
if action == "build_lane":
result = _run_ml(["build", "lane"])
_auto_snapshot("lane")
@@ -229,10 +258,7 @@ def execute_action(action: str, params: dict[str, Any]) -> dict[str, Any]:
"export_ls_to_yolo: 无有效标注可导出 (written=0); "
f"skipped_empty={conv.get('skipped_empty')} missing_ann={conv.get('missing_ann')}"
)
argv = ["build", "dms", task, "--pack", pack, "--batch", batch]
result = _run_ml(argv)
result["export_convert"] = conv
return result
return {"ok": True, "stdout": json.dumps(conv, ensure_ascii=False), "stderr": "", "export_convert": conv}
if row.get("project") == "lane" and export == "lane_gt_txt":
scripts_dir = WORKSPACE / "datasets" / "lane" / "scripts"
if str(scripts_dir) not in sys.path:
@@ -250,10 +276,22 @@ def execute_action(action: str, params: dict[str, Any]) -> dict[str, Any]:
"export_ls_to_lane_gt: 无有效标注可导出 (written=0); "
f"skipped_empty={conv.get('skipped_empty')} missing_ann={conv.get('missing_ann')}"
)
argv = ["build", "lane"]
result = _run_ml(argv)
result["export_convert"] = conv
return result
return {"ok": True, "stdout": json.dumps(conv, ensure_ascii=False), "stderr": "", "export_convert": conv}
if row.get("project") == "adas" and export == "cvat_cuboid":
from as_platform.labeling.export_cuboid_batch import export_batch as export_cuboid_batch
with session_scope() as db:
camp = db.get(LabelingCampaign, campaign_id)
if not camp:
raise ValueError("campaign not found")
batch_dir = resolve_campaign_batch_dir(camp)
conv = export_cuboid_batch(batch_dir)
if conv.get("written", 0) == 0:
raise ValueError(
"export_cuboid_batch: 无有效 cuboid 可导出 (written=0); "
f"skipped_empty={conv.get('skipped_empty')} missing_ann={conv.get('missing_ann')}"
)
return {"ok": True, "stdout": json.dumps(conv, ensure_ascii=False), "stderr": "", "export_convert": conv}
return {
"ok": True,
"stdout": json.dumps({"export": export, "campaign": row}, ensure_ascii=False),

View File

@@ -1,6 +1,7 @@
"""同步 inbox/sources 批次 batch.meta.yaml 的 stage与 Campaign 状态一致。"""
from __future__ import annotations
import json
from pathlib import Path
from as_platform.data.batch import read_meta, write_meta
@@ -18,6 +19,22 @@ def batch_has_yolo_labels(batch_dir: Path) -> bool:
return False
def batch_has_cuboid_labels(batch_dir: Path) -> bool:
"""批次是否已有导出的 ADAS quaternion_json含非空 detections"""
qdir = batch_dir / "labels" / "quaternion_json"
if not qdir.is_dir():
return False
for p in qdir.glob("*.json"):
try:
data = json.loads(p.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
continue
dets = data.get("detections") or []
if isinstance(dets, list) and len(dets) > 0:
return True
return False
def batch_has_lane_labels(batch_dir: Path) -> bool:
"""批次是否已有 UFLD mask 清单list/train_gt.txt + annotations/*.png"""
list_path = batch_dir / "list" / "train_gt.txt"
@@ -29,6 +46,14 @@ def batch_has_lane_labels(batch_dir: Path) -> bool:
return any(ann_dir.rglob("*.png"))
def _batch_has_export_labels(project: str, batch_dir: Path) -> bool:
if project == "lane":
return batch_has_lane_labels(batch_dir)
if project == "adas":
return batch_has_cuboid_labels(batch_dir)
return batch_has_yolo_labels(batch_dir)
def update_campaign_batch_meta_stage(camp: LabelingCampaign, stage: str) -> bool:
try:
batch_dir = resolve_campaign_batch_dir(camp)
@@ -56,8 +81,23 @@ def update_campaign_batch_meta_stage_by_id(campaign_id: str, stage: str) -> bool
return update_campaign_batch_meta_stage(camp, stage)
def _advance_campaign_stage(campaign_id: str, stage: str) -> None:
with session_scope() as db:
camp = db.get(LabelingCampaign, str(campaign_id))
if not camp:
return
camp.status = stage
db.flush()
update_campaign_batch_meta_stage(camp, stage)
def _batch_has_calib(batch_dir: Path) -> bool:
calib = batch_dir / "calib"
return calib.is_dir() and bool(list(calib.glob("*.yaml")) + list(calib.glob("*.yml")))
def on_labeling_export_job_succeeded(job: dict) -> None:
"""导出 Job 成功且批次已有训练标签时进入 returned入库)。"""
"""导出 Job 成功且批次已有训练标签时进入 returned build)。"""
if job.get("action") != "labeling_export":
return
params = job.get("params") or {}
@@ -72,10 +112,46 @@ def on_labeling_export_job_succeeded(job: dict) -> None:
batch_dir = resolve_campaign_batch_dir(camp)
except Exception:
return
has_labels = (
batch_has_lane_labels(batch_dir)
if camp.project == "lane"
else batch_has_yolo_labels(batch_dir)
)
if has_labels:
update_campaign_batch_meta_stage_by_id(str(cid), "returned")
project = camp.project or "dms"
if _batch_has_export_labels(project, batch_dir):
_advance_campaign_stage(str(cid), "returned")
if project == "adas" and _batch_has_calib(batch_dir):
from as_platform.jobs.queue import enqueue_job
enqueue_job(
"cuboid_fit_3d",
{"campaign_id": str(cid)},
async_run=True,
)
def on_build_job_succeeded(job: dict) -> None:
"""build Job 成功后将批次晋升 ingested。"""
action = job.get("action")
if action not in ("build_dms", "build_adas", "build_lane"):
return
params = job.get("params") or {}
batch = params.get("batch")
if not batch:
return
project = params.get("project")
if not project:
if action == "build_adas":
project = "adas"
elif action == "build_lane":
project = "lane"
else:
project = "dms"
task = params.get("task")
with session_scope() as db:
q = db.query(LabelingCampaign).filter(LabelingCampaign.batch == str(batch))
if task:
q = q.filter(LabelingCampaign.task == str(task))
if project:
q = q.filter(LabelingCampaign.project == str(project))
camp = q.order_by(LabelingCampaign.created_at.desc()).first()
if not camp:
return
camp.status = "ingested"
db.flush()
update_campaign_batch_meta_stage(camp, "ingested")

View File

@@ -0,0 +1,74 @@
"""ADAS class_id 映射BK2/MOON 单源)。"""
from __future__ import annotations
from pathlib import Path
from typing import Any
import yaml
from as_platform.config import WORKSPACE
_ADAS_REGISTRY = WORKSPACE / "datasets" / "adas" / "adas.registry.yaml"
_LABELING_REGISTRY = WORKSPACE / "datasets" / "labeling.registry.yaml"
def load_adas_class_names() -> list[str]:
if _ADAS_REGISTRY.is_file():
reg = yaml.safe_load(_ADAS_REGISTRY.read_text(encoding="utf-8")) or {}
names = (reg.get("classes") or {}).get("names")
if names:
return [str(n) for n in names]
if _LABELING_REGISTRY.is_file():
reg = yaml.safe_load(_LABELING_REGISTRY.read_text(encoding="utf-8")) or {}
labels = (reg.get("profiles") or {}).get("cuboid_7cls", {}).get("cvat_labels")
if labels:
return [str(n) for n in labels]
from as_platform.labeling.format_converter import CUBOID_7CLS_NAMES
return list(CUBOID_7CLS_NAMES)
def class_name_to_id(name: str, class_map: dict[str, int] | None = None) -> int | None:
cmap = class_map or {n: i for i, n in enumerate(load_adas_class_names())}
if name in cmap:
return cmap[name]
low = name.lower()
for k, v in cmap.items():
if k.lower() == low:
return v
return None
def build_class_map(names: list[str] | None = None) -> dict[str, int]:
return {str(n): idx for idx, n in enumerate(names or load_adas_class_names())}
def remap_class_id(old_names: list[str], new_names: list[str], class_id: int) -> int:
if class_id < 0 or class_id >= len(old_names):
return class_id
label = old_names[class_id]
new_id = build_class_map(new_names).get(label)
if new_id is None:
for k, v in build_class_map(new_names).items():
if k.lower() == label.lower():
return v
return new_id if new_id is not None else class_id
def normalize_detection_class(det: dict[str, Any], class_map: dict[str, int] | None = None) -> dict[str, Any]:
cmap = class_map or build_class_map()
name = str(det.get("class_name") or "")
cid = det.get("class_id")
if name:
mapped = class_name_to_id(name, cmap)
if mapped is not None:
det = dict(det)
det["class_id"] = mapped
det["class_name"] = name
elif cid is not None:
names = list(cmap.keys())
idx = int(cid)
if 0 <= idx < len(names):
det = dict(det)
det["class_name"] = names[idx]
return det

View File

@@ -0,0 +1,174 @@
"""ls_annotations cuboid → labels/quaternion_json/*.jsonADAS MOON-3D 兼容格式)。"""
from __future__ import annotations
import hashlib
import json
from pathlib import Path
from typing import Any
import yaml
from as_platform.labeling.class_map import build_class_map, load_adas_class_names
from as_platform.labeling.format_converter import cuboid_item_to_detection
IMG_EXTS = {".jpg", ".jpeg", ".png", ".bmp", ".webp", ".JPG", ".JPEG", ".PNG"}
ANNOTATIONS_DIRNAME = "ls_annotations"
def _load_cuboid_class_map() -> dict[str, int]:
return build_class_map(load_adas_class_names())
def _task_id_for_image(image_path: Path, batch_dir: Path) -> str:
try:
rel = image_path.relative_to(batch_dir)
stem = rel.as_posix()
except ValueError:
stem = image_path.name
return hashlib.sha256(stem.encode()).hexdigest()[:16]
def _iter_batch_images(batch_dir: Path) -> list[Path]:
if not batch_dir.is_dir():
return []
candidates: list[Path] = []
search_roots = [batch_dir / "images", batch_dir / "images" / "train", batch_dir]
seen: set[str] = set()
for root in search_roots:
if not root.is_dir():
continue
for p in sorted(root.rglob("*")):
if not p.is_file() or p.suffix not in IMG_EXTS:
continue
key = str(p.resolve())
if key in seen:
continue
seen.add(key)
candidates.append(p.resolve())
return candidates
def _extract_result_regions(data: dict[str, Any]) -> list[dict[str, Any]]:
result = data.get("result")
if isinstance(result, list) and result:
return result
annotations = data.get("annotations")
if isinstance(annotations, list) and annotations:
first = annotations[0]
if isinstance(first, dict) and isinstance(first.get("result"), list):
return first["result"]
return []
def _find_calib(batch_dir: Path) -> tuple[Path | None, list[list[float]] | None, list[int] | None]:
calib_dir = batch_dir / "calib"
if not calib_dir.is_dir():
return None, None, None
yaml_files = sorted(calib_dir.glob("*.yaml")) + sorted(calib_dir.glob("*.yml"))
if not yaml_files:
return None, None, None
path = yaml_files[0]
try:
data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
except (OSError, yaml.YAMLError):
return path, None, None
K = data.get("K")
image_size = data.get("image_size")
if K and isinstance(K, list) and len(K) == 3:
return path, K, list(image_size) if image_size else None
fx = data.get("fx")
fy = data.get("fy")
cx = data.get("cx")
cy = data.get("cy")
if fx is not None and fy is not None and cx is not None and cy is not None:
K = [[float(fx), 0.0, float(cx)], [0.0, float(fy), float(cy)], [0.0, 0.0, 1.0]]
return path, K, list(image_size) if image_size else None
return path, None, list(image_size) if image_size else None
def _resolve_image_for_ann(data: dict[str, Any], batch_dir: Path, task_id: str) -> Path | None:
image_name = data.get("image")
if image_name:
for root in (batch_dir / "images", batch_dir):
candidate = root / str(image_name)
if candidate.is_file():
return candidate
for p in root.rglob(str(image_name)):
if p.is_file():
return p
for image_path in _iter_batch_images(batch_dir):
if _task_id_for_image(image_path, batch_dir) == task_id:
return image_path
return None
def export_batch(batch_dir: Path) -> dict[str, Any]:
"""导出 cuboid ls_annotations → quaternion_json。"""
batch_dir = batch_dir.resolve()
class_map = _load_cuboid_class_map()
calib_path, K, calib_size = _find_calib(batch_dir)
ann_dir = batch_dir / "labels" / ANNOTATIONS_DIRNAME
out_dir = batch_dir / "labels" / "quaternion_json"
out_dir.mkdir(parents=True, exist_ok=True)
written = 0
skipped_empty = 0
missing_ann = 0
for ann_path in sorted(ann_dir.glob("*.json")):
task_id = ann_path.stem
try:
data = json.loads(ann_path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
missing_ann += 1
continue
regions = _extract_result_regions(data)
cuboids = [r for r in regions if r.get("type") == "cuboid"]
if not cuboids:
skipped_empty += 1
continue
image_path = _resolve_image_for_ann(data, batch_dir, task_id)
if not image_path:
missing_ann += 1
continue
detections: list[dict[str, Any]] = []
for item in cuboids:
det = cuboid_item_to_detection(item, class_map, K=K)
if det:
detections.append(det)
if not detections:
skipped_empty += 1
continue
img_w = int((cuboids[0].get("original_width") or (calib_size or [1920, 1080])[0]))
img_h = int((cuboids[0].get("original_height") or (calib_size or [1920, 1080])[1]))
payload: dict[str, Any] = {
"image": str(image_path),
"image_stem": image_path.stem,
"image_size": [img_w, img_h],
"coordinate_frame": "opencv_camera",
"boxes3d_format": "center_3d + dimensions_wlh + quaternion_wxyz",
"text_prompts": load_adas_class_names(),
"num_detections": len(detections),
"detections": detections,
}
if K:
payload["K"] = K
payload["k_source"] = calib_path.name if calib_path else "fixed_calib"
else:
payload["k_source"] = "missing_calib"
out_path = out_dir / f"{image_path.stem}.json"
out_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
written += 1
return {
"written": written,
"skipped_empty": skipped_empty,
"missing_ann": missing_ann,
"missing_calib": calib_path is None or K is None,
"calib": str(calib_path) if calib_path else None,
}

View File

@@ -0,0 +1,95 @@
"""Batch-level cuboid 3D fit for quaternion_json."""
from __future__ import annotations
import hashlib
import json
from pathlib import Path
from typing import Any
IMG_EXTS = {".jpg", ".jpeg", ".png", ".bmp", ".webp"}
def _task_id_for_image(image_path: Path, batch_dir: Path) -> str:
try:
rel = image_path.relative_to(batch_dir)
stem = rel.as_posix()
except ValueError:
stem = image_path.name
return hashlib.sha256(stem.encode()).hexdigest()[:16]
def _load_ls_cuboid_points(batch_dir: Path, stem: str) -> list[list[float]]:
ann_dir = batch_dir / "labels" / "ls_annotations"
if not ann_dir.is_dir():
return []
for p in ann_dir.glob("*.json"):
try:
data = json.loads(p.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
continue
img = str(data.get("image") or "")
if stem in img or p.stem:
regions = data.get("result") or []
pts_list = []
for r in regions:
if r.get("type") != "cuboid":
continue
pts = list(r.get("points") or [])
if len(pts) >= 16:
pts_list.append(pts[:16])
if pts_list:
return pts_list
return []
def fit_batch(batch_dir: Path) -> dict[str, Any]:
from algorithms.adas_mono3d.fit_cuboid import fit_cuboid_detection
batch_dir = batch_dir.resolve()
qdir = batch_dir / "labels" / "quaternion_json"
if not qdir.is_dir():
raise ValueError(f"missing {qdir}")
updated = 0
fit_ok = 0
total = 0
for p in sorted(qdir.glob("*.json")):
data = json.loads(p.read_text(encoding="utf-8"))
K = data.get("K")
if not K:
continue
stem = data.get("image_stem") or p.stem
cuboid_pts_list = _load_ls_cuboid_points(batch_dir, stem)
new_dets = []
for i, det in enumerate(data.get("detections") or []):
det = dict(det)
if det.get("fit_ok"):
new_dets.append(det)
total += 1
fit_ok += 1
continue
class_name = str(det.get("class_name") or "car")
points = cuboid_pts_list[i] if i < len(cuboid_pts_list) else None
if not points:
box = det.get("box2d_xyxy") or []
if len(box) >= 4:
x1, y1, x2, y2 = box[:4]
points = [x1, y1, x2, y1, x1, y2, x2, y2, x1, y1, x2, y1, x1, y2, x2, y2]
if points:
fitted = fit_cuboid_detection(points, K, class_name)
det.update({k: v for k, v in fitted.items() if k != "box2d_xyxy" or "box2d_xyxy" not in det})
new_dets.append(det)
total += 1
if det.get("fit_ok"):
fit_ok += 1
data["detections"] = new_dets
data["num_detections"] = len(new_dets)
p.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
updated += 1
return {
"updated_files": updated,
"detections": total,
"fit_ok": fit_ok,
"fit_ok_ratio": fit_ok / max(total, 1),
}

View File

@@ -491,6 +491,71 @@ def convert_cvat_kitti_export_to_hsap(kitti_data: bytes, output_dir: Path) -> in
return count
# ═══════════════════════════════════════════════════════
# CVAT cuboid 16pt → HSAP quaternion_json detection (MVP)
# ═══════════════════════════════════════════════════════
CUBOID_7CLS_NAMES = [
"pedestrian",
"car",
"truck",
"bus",
"motorcycle",
"tricycle",
"traffic cone",
]
def cuboid_points_to_box2d(points: list[float]) -> list[float] | None:
"""从 CVAT cuboid 16 点8 个 x,y 对)计算 axis-aligned 2D bbox。"""
if len(points) < 16:
return None
xs = [float(points[i]) for i in range(0, 16, 2)]
ys = [float(points[i]) for i in range(1, 16, 2)]
return [min(xs), min(ys), max(xs), max(ys)]
def cuboid_item_to_detection(
item: dict[str, Any],
class_map: dict[str, int],
*,
K: list[list[float]] | None = None,
) -> dict[str, Any] | None:
"""ls_annotations cuboid 条目 → quaternion_json detectionMVP2D bbox + 可选 3D 占位)。"""
label = str(item.get("label") or "")
class_id = class_map.get(label)
if class_id is None:
for name, cid in class_map.items():
if name.lower() == label.lower():
class_id = cid
break
if class_id is None:
return None
points = item.get("points") or []
if len(points) < 16:
for key in (
"xtl1", "ytl1", "xtr1", "ytr1", "xbl1", "ybl1", "xbr1", "ybr1",
"xtl2", "ytl2", "xtr2", "ytr2", "xbl2", "ybl2", "xbr2", "ybr2",
):
if key in item:
points.append(float(item[key]))
box2d = cuboid_points_to_box2d(points)
if not box2d:
return None
det: dict[str, Any] = {
"class_id": class_id,
"class_name": label,
"score": 1.0,
"box2d_xyxy": box2d,
"fit_ok": False,
}
if K:
det["K_used"] = True
return det
# ═══════════════════════════════════════════════════════
# ADAS 3D Quaternion JSON → CVAT cuboid XML
# ═══════════════════════════════════════════════════════

View File

@@ -18,6 +18,7 @@ from as_platform.labeling.batch_stage import (
on_labeling_export_job_succeeded,
update_campaign_batch_meta_stage,
)
from as_platform.labeling.stage import effective_stage, matches_stage_filter
from as_platform.labeling.scope import (
enrich_batch_labels,
format_scope_key,
@@ -120,11 +121,14 @@ def list_labeling_batches(
def _append(b: dict[str, Any]) -> None:
if b.get("registry_only"):
return
if stage and b.get("stage") != stage:
raw_stage = b.get("stage")
eff = effective_stage(raw_stage)
if stage and not matches_stage_filter(raw_stage, stage):
return
if b.get("stage") not in allowed_stages:
if eff not in allowed_stages and raw_stage not in allowed_stages:
return
row = enrich_batch_labels(b, reg)
row["stage"] = eff or raw_stage
cid = _campaign_id(
row["project"], row.get("task") or "", row.get("mode"), row["batch"], row.get("location") or "inbox"
)
@@ -470,6 +474,48 @@ def trigger_labeling_export(campaign_id: str) -> dict[str, Any]:
return {"ok": True, "job": job, "export_job": ej, "export_default": row.get("export_default")}
def get_batch_export_stats(campaign_id: str) -> dict[str, Any]:
from as_platform.labeling.annotate import resolve_campaign_batch_dir
from as_platform.data.promote.validate.adas_cuboid import validate_adas_cuboid_batch
from as_platform.labeling.batch_stage import batch_has_cuboid_labels, batch_has_yolo_labels
with session_scope() as db:
camp = db.get(LabelingCampaign, campaign_id)
if not camp:
raise FileNotFoundError("campaign not found")
project = camp.project
batch_dir = resolve_campaign_batch_dir(camp)
if project == "adas":
_errors, warnings, stats = validate_adas_cuboid_batch(batch_dir, allow_partial_3d=True)
calib = (batch_dir / "calib").is_dir() and bool(list((batch_dir / "calib").glob("*.yaml")))
return {
"project": "adas",
"campaign_id": campaign_id,
"pack_default": "adas_moon3d_v1",
"quaternion_files": stats.get("quaternion_files", 0),
"fit_ok_ratio": stats.get("fit_ok_ratio", 0),
"missing_calib": not calib,
"stats": stats,
"warnings": warnings,
}
return {
"project": project,
"campaign_id": campaign_id,
"has_yolo": batch_has_yolo_labels(batch_dir),
"has_cuboid": batch_has_cuboid_labels(batch_dir),
}
def trigger_cuboid_fit(campaign_id: str) -> dict[str, Any]:
row = get_campaign(campaign_id)
if not row:
raise FileNotFoundError("campaign not found")
if row.get("project") != "adas":
raise ValueError("cuboid_fit_3d 仅适用于 ADAS")
job = enqueue_job("cuboid_fit_3d", {"campaign_id": campaign_id}, async_run=True)
return {"ok": True, "job": job}
# ═══════════════════════════════════════════════════════
# CVAT 集成辅助
# ═══════════════════════════════════════════════════════

View File

@@ -0,0 +1,29 @@
"""标注批次 stage 读时归一化(兼容旧 pipeline"""
from __future__ import annotations
STAGE_ALIASES: dict[str, str] = {
"review_approved": "labeling_submitted",
}
CANONICAL_STAGES = (
"raw_pool",
"out_for_labeling",
"in_review",
"review_rejected",
"labeling_submitted",
"returned",
"ingested",
)
def effective_stage(stage: str | None) -> str | None:
if not stage:
return stage
return STAGE_ALIASES.get(stage, stage)
def matches_stage_filter(batch_stage: str | None, filter_stage: str | None) -> bool:
if not filter_stage:
return True
eff = effective_stage(batch_stage)
return eff == filter_stage or batch_stage == filter_stage

View File

@@ -0,0 +1,213 @@
#!/usr/bin/env python3
"""Unified Ingest SDK 单元测试(无 pytest 依赖)。"""
from __future__ import annotations
import json
import shutil
import sys
import tempfile
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
PLATFORM = ROOT / "platform"
if str(PLATFORM) not in sys.path:
sys.path.insert(0, str(PLATFORM))
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
def test_stage_aliases() -> None:
from as_platform.labeling.stage import effective_stage, matches_stage_filter
assert effective_stage("review_approved") == "labeling_submitted"
assert effective_stage("returned") == "returned"
assert matches_stage_filter("review_approved", "labeling_submitted")
assert not matches_stage_filter("raw_pool", "returned")
def test_bk2_class_map() -> None:
from as_platform.labeling.class_map import (
build_class_map,
load_adas_class_names,
normalize_detection_class,
remap_class_id,
)
names = load_adas_class_names()
assert names[0] == "pedestrian"
assert names[1] == "car"
cmap = build_class_map(names)
assert cmap["car"] == 1
assert cmap["pedestrian"] == 0
old = ["car", "pedestrian", "truck", "bus", "motorcycle", "tricycle", "traffic cone"]
assert remap_class_id(old, names, 0) == 1 # car was 0, now 1
det = normalize_detection_class({"class_name": "car", "class_id": 99})
assert det["class_id"] == 1
def test_validate_adas_cuboid() -> None:
from as_platform.data.promote.validate.adas_cuboid import validate_adas_cuboid_batch
from as_platform.labeling.class_map import load_adas_class_names
with tempfile.TemporaryDirectory() as td:
batch = Path(td)
qdir = batch / "labels" / "quaternion_json"
qdir.mkdir(parents=True)
names = load_adas_class_names()
good = {
"detections": [{"class_id": 1, "class_name": "car", "fit_ok": False}],
"text_prompts": names,
"K": [[1000, 0, 960], [0, 1000, 540], [0, 0, 1]],
}
(qdir / "a.json").write_text(json.dumps(good), encoding="utf-8")
(qdir / "empty.json").write_text(json.dumps({"detections": []}), encoding="utf-8")
(batch / "calib").mkdir()
(batch / "calib" / "cam.yaml").write_text("K: []\n", encoding="utf-8")
errors, warnings, stats = validate_adas_cuboid_batch(batch, allow_partial_3d=True)
assert not errors, errors
assert stats["files_with_detections"] == 1
assert any("empty" in w for w in warnings)
def test_fit_cuboid_detection() -> None:
from algorithms.adas_mono3d.fit_cuboid import cuboid_points_to_box2d, fit_cuboid_detection
pts = [770.0, 347.0, 834.0, 347.0, 772.0, 423.0, 835.0, 423.0,
806.0, 357.0, 861.0, 357.0, 807.0, 422.0, 862.0, 422.0]
box = cuboid_points_to_box2d(pts)
assert box is not None
assert box[0] < box[2] and box[1] < box[3]
K = [[1189.7, 0, 1007.5], [0, 1189.7, 517.5], [0, 0, 1]]
out = fit_cuboid_detection(pts, K, "car")
assert "center_3d" in out
assert "dimensions_wlh" in out
assert "quaternion_wxyz" in out
assert len(out["quaternion_wxyz"]) == 4
def test_export_cuboid_batch_class_id() -> None:
from as_platform.labeling.export_cuboid_batch import export_batch
with tempfile.TemporaryDirectory() as td:
batch = Path(td)
(batch / "images").mkdir()
img = batch / "images" / "frame1.jpg"
img.write_bytes(b"\xff\xd8\xff")
calib = batch / "calib" / "cam0.yaml"
calib.parent.mkdir()
calib.write_text(
"K:\n - [1000, 0, 960]\n - [0, 1000, 540]\n - [0, 0, 1]\n",
encoding="utf-8",
)
ann_dir = batch / "labels" / "ls_annotations"
ann_dir.mkdir(parents=True)
ann = {
"image": "frame1.jpg",
"result": [{
"type": "cuboid",
"label": "car",
"points": [770.0, 347.0, 834.0, 347.0, 772.0, 423.0, 835.0, 423.0,
806.0, 357.0, 861.0, 357.0, 807.0, 422.0, 862.0, 422.0],
"original_width": 1920,
"original_height": 1080,
}],
}
import hashlib
tid = hashlib.sha256(b"images/frame1.jpg").hexdigest()[:16]
(ann_dir / f"{tid}.json").write_text(json.dumps(ann), encoding="utf-8")
result = export_batch(batch)
assert result["written"] == 1
qjson = batch / "labels" / "quaternion_json" / "frame1.json"
assert qjson.is_file()
data = json.loads(qjson.read_text())
assert data["text_prompts"][0] == "pedestrian"
assert data["detections"][0]["class_id"] == 1
assert data["detections"][0]["class_name"] == "car"
def test_refresh_adas_lists() -> None:
from as_platform.data.promote.manifest import refresh_adas_lists
with tempfile.TemporaryDirectory() as td:
pack_root = Path(td) / "packs" / "test_pack"
src = pack_root / "sources" / "batch_a" / "labels" / "quaternion_json"
src.mkdir(parents=True)
(src / "img1.json").write_text('{"detections":[{}]}', encoding="utf-8")
(src / "img2.json").write_text('{"detections":[{}]}', encoding="utf-8")
wf = {
"projects": {
"adas": {
"root": str(Path(td)),
"registry": "adas.registry.yaml",
}
}
}
(Path(td) / "adas.registry.yaml").write_text("split:\n val_ratio: 0.5\n", encoding="utf-8")
out = refresh_adas_lists(wf, pack="test_pack")
train = Path(out["train_list"]).read_text().strip().splitlines()
val = Path(out["val_list"]).read_text().strip().splitlines()
assert len(train) + len(val) == 2
assert Path(out["pack_index"]).is_file()
def test_promote_adas_dry_run() -> None:
from as_platform.data.promote.adas_cuboid import AdasCuboidPromoteAdapter
from as_platform.data.promote.base import PromoteContext
from as_platform.labeling.class_map import load_adas_class_names
with tempfile.TemporaryDirectory() as td:
batch = Path(td) / "inbox" / "cuboid_7cls" / "b1"
qdir = batch / "labels" / "quaternion_json"
qdir.mkdir(parents=True)
names = load_adas_class_names()
payload = {
"detections": [{"class_id": 1, "class_name": "car"}],
"text_prompts": names,
"K": [[1000, 0, 960], [0, 1000, 540], [0, 0, 1]],
}
(qdir / "f.json").write_text(json.dumps(payload), encoding="utf-8")
(batch / "images").mkdir()
(batch / "images" / "f.jpg").write_bytes(b"x")
root = Path(td)
ctx = PromoteContext(
project="adas",
task="cuboid_7cls",
batch="b1",
pack="test_pack",
batch_dir=batch,
project_root=root,
dry_run=True,
)
adapter = AdasCuboidPromoteAdapter()
assert adapter.validate(ctx) == []
result = adapter.promote(ctx)
assert result.ok
assert result.detail.get("dry_run") is True
def main() -> None:
tests = [
test_stage_aliases,
test_bk2_class_map,
test_validate_adas_cuboid,
test_fit_cuboid_detection,
test_export_cuboid_batch_class_id,
test_refresh_adas_lists,
test_promote_adas_dry_run,
]
for fn in tests:
fn()
print(f"OK {fn.__name__}")
print(f"ALL {len(tests)} PASSED")
if __name__ == "__main__":
main()

View File

@@ -162,6 +162,12 @@ export const hsapApi = {
labelingExport: (campaignId: string) =>
postJson<{ ok: boolean; job?: { id: string } }>(`${API_BASE}/api/v1/labeling/campaigns/${campaignId}/export`),
labelingExportStats: (campaignId: string) =>
fetchJson<Record<string, unknown>>(`${API_BASE}/api/v1/labeling/campaigns/${campaignId}/export-stats`),
cuboidFit: (campaignId: string) =>
postJson<Record<string, unknown>>(`${API_BASE}/api/v1/labeling/campaigns/${campaignId}/cuboid-fit`),
submitLabelingCampaign: (campaignId: string) =>
postJson<Record<string, unknown>>(`${API_BASE}/api/v1/labeling/campaigns/${campaignId}/submit`),

View File

@@ -339,7 +339,13 @@ export const CampaignsPage: React.FC = () => {
</Link>
<button
onClick={() => handleExport(b.campaign_id!)}
className="inline-flex items-center gap-1 px-3 py-1.5 text-xs font-medium rounded-lg bg-gray-50 text-gray-600 hover:bg-gray-100 transition-colors"
disabled={!["labeling_submitted", "returned"].includes(b.stage || "")}
title={!["labeling_submitted", "returned"].includes(b.stage || "") ? "质检通过后才可导出" : undefined}
className={`inline-flex items-center gap-1 px-3 py-1.5 text-xs font-medium rounded-lg transition-colors ${
["labeling_submitted", "returned"].includes(b.stage || "")
? "bg-gray-50 text-gray-600 hover:bg-gray-100"
: "bg-gray-50 text-gray-300 cursor-not-allowed"
}`}
>
📤
</button>
@@ -347,7 +353,7 @@ export const CampaignsPage: React.FC = () => {
onClick={() => handleSubmit(b.campaign_id!)}
className="inline-flex items-center gap-1 px-3 py-1.5 text-xs font-medium rounded-lg bg-green-50 text-green-700 hover:bg-green-100 transition-colors"
>
</button>
<button
onClick={() => toggleExpand(b.campaign_id!)}

View File

@@ -10,7 +10,11 @@ export const ExportPage: React.FC = () => {
const [batches, setBatches] = useState<LabelingBatchRow[]>([]);
const [loading, setLoading] = useState(true);
const [error, setError] = useState<string | null>(null);
const [info, setInfo] = useState<string | null>(null);
const [importingId, setImportingId] = useState<string | null>(null);
const [statsMap, setStatsMap] = useState<Record<string, Record<string, unknown>>>({});
const [buildingId, setBuildingId] = useState<string | null>(null);
const [fittingId, setFittingId] = useState<string | null>(null);
const [search, setSearch] = useState("");
const [stageFilter, setStageFilter] = useState("");
@@ -33,6 +37,20 @@ export const ExportPage: React.FC = () => {
if (returned.status === "fulfilled") results.push(...((returned.value.items || []) as LabelingBatchRow[]));
if (submitted.status === "rejected" && returned.status === "rejected") setError(String(submitted.reason));
setBatches(results);
const adasReturned = results.filter((b) => b.stage === "returned" && b.project === "adas" && b.campaign_id);
if (adasReturned.length) {
const entries = await Promise.allSettled(
adasReturned.map(async (b) => {
const s = await hsapApi.labelingExportStats(b.campaign_id!);
return [b.campaign_id!, s] as const;
}),
);
const map: Record<string, Record<string, unknown>> = {};
for (const e of entries) {
if (e.status === "fulfilled") map[e.value[0]] = e.value[1];
}
setStatsMap(map);
}
} catch (e) { setError(String(e)); }
setLoading(false);
}, []);
@@ -40,10 +58,41 @@ export const ExportPage: React.FC = () => {
useEffect(() => { load(); }, [load]);
const handleExport = async (campaignId: string) => {
try { await hsapApi.labelingExport(campaignId); load(); }
try { await hsapApi.labelingExport(campaignId); setInfo("导出任务已提交"); load(); }
catch (e) { setError(String(e)); }
};
const handleCuboidFit = async (campaignId: string) => {
setFittingId(campaignId);
try {
await hsapApi.cuboidFit(campaignId);
setInfo("3D 拟合任务已提交");
load();
} catch (e) { setError(String(e)); }
setFittingId(null);
};
const handleSubmitBuild = async (b: LabelingBatchRow) => {
if (!b.task || !b.batch) return;
setBuildingId(b.campaign_id || b.batch);
setError(null);
try {
await hsapApi.submitBuildBatch({
project: b.project || "dms",
task: b.task,
batch: b.batch,
pack: b.pack || (b.project === "adas" ? "adas_moon3d_v1" : "dms_v2"),
location: b.location || "inbox",
note: `入库 ${b.batch}`,
});
setInfo("build 已提交至审核队列");
load();
} catch (e) {
setError(String(e));
}
setBuildingId(null);
};
const handleImportVendor = async (campaignId: string) => {
const input = document.createElement("input");
input.type = "file"; input.accept = ".zip";
@@ -64,9 +113,11 @@ export const ExportPage: React.FC = () => {
<div className="page-container">
<div className="page-header">
<h1></h1>
<p></p>
<p>build </p>
</div>
{info && <div className="bg-green-50 border border-green-200 rounded p-3 mb-4 text-sm text-green-700">{info}</div>}
<div className="bg-white rounded-xl border border-gray-200 p-3 mb-4">
<div className="flex items-center gap-3 flex-wrap">
<div className="flex-1 min-w-[200px] relative">
@@ -77,7 +128,7 @@ export const ExportPage: React.FC = () => {
placeholder="搜索批次/任务..." value={search} onChange={(e) => setSearch(e.target.value)} />
</div>
<div className="flex gap-1.5">
{["全部", "待导出", "待入库"].map((label, i) => {
{["全部", "待导出", "待 build"].map((label, i) => {
const val = i === 0 ? "" : ["labeling_submitted", "returned"][i - 1];
return <button key={val} onClick={() => setStageFilter(val)} className={`px-3 py-1.5 rounded-lg text-xs font-medium transition-colors ${stageFilter === val ? "bg-blue-600 text-white" : "bg-gray-100 text-gray-600 hover:bg-gray-200"}`}>{label}</button>;
})}
@@ -86,33 +137,29 @@ export const ExportPage: React.FC = () => {
</div>
</div>
{/* Workflow guide */}
<div className="card mb-4">
<div className="card-header"></div>
<div className="text-sm text-gray-600 space-y-2">
<div className="flex items-center gap-3">
<span className="w-8 h-8 rounded-full bg-blue-100 text-blue-700 flex items-center justify-center text-xs font-bold">1</span>
<span><strong></strong> Campaign "提交批次"</span>
<span><strong></strong> </span>
</div>
<div className="flex items-center gap-3">
<span className="w-8 h-8 rounded-full bg-blue-100 text-blue-700 flex items-center justify-center text-xs font-bold">2</span>
<span>
<strong></strong> "执行导出" YOLO
{hasData && <span className="text-gray-400"></span>}
</span>
<span><strong></strong> </span>
</div>
<div className="flex items-center gap-3">
<span className="w-8 h-8 rounded-full bg-orange-100 text-orange-700 flex items-center justify-center text-xs font-bold">3</span>
<span>
<strong></strong> "导入供应商" ZIP
<span className="block text-xs text-gray-400 mt-0.5">ZIP .txt YOLO </span>
<strong></strong> CVAT DMSYOLOADASquaternion_json
{hasData && <span className="text-gray-400"></span>}
</span>
</div>
<div className="flex items-center gap-3">
<span className="w-8 h-8 rounded-full bg-green-100 text-green-700 flex items-center justify-center text-xs font-bold">4</span>
<span>
<strong> build</strong> <Badge variant="success" size="small"></Badge> build
<span className="block text-xs text-gray-400 mt-0.5">build </span>
<strong> build</strong> <Badge variant="warning" size="small"> build</Badge> build <Badge variant="success" size="small"></Badge>
<span className="block text-xs text-gray-400 mt-0.5"> buildingested </span>
</span>
</div>
</div>
@@ -127,10 +174,17 @@ export const ExportPage: React.FC = () => {
<div className="flex-1 min-w-0">
<div className="flex items-center gap-2">
<span className="font-semibold text-sm">{b.batch}</span>
<span className="text-xs text-gray-400">{b.task || "—"}</span>
<Badge variant={b.stage === "returned" ? "success" : "warning"}>{b.stage === "labeling_submitted" ? "待导出" : "待入库"}</Badge>
<span className="text-xs text-gray-400">{b.project}/{b.task || "—"}</span>
<Badge variant={b.stage === "returned" ? "warning" : "warning"}>{b.stage === "labeling_submitted" ? "待导出" : "待 build"}</Badge>
</div>
<div className="text-xs text-gray-400 font-mono mt-1">{b.campaign_id?.slice(0, 16) || "—"}</div>
{b.stage === "returned" && b.project === "adas" && b.campaign_id && statsMap[b.campaign_id] && (
<div className="text-xs text-gray-500 mt-1">
quaternion: {String(statsMap[b.campaign_id].quaternion_files ?? "—")} ·
fit_ok: {((Number(statsMap[b.campaign_id].fit_ok_ratio) || 0) * 100).toFixed(0)}% ·
pack: adas_moon3d_v1
</div>
)}
</div>
<div className="flex items-center gap-1.5 shrink-0">
{b.campaign_id && b.stage === "labeling_submitted" && (
@@ -139,7 +193,24 @@ export const ExportPage: React.FC = () => {
<Button size="small" variant="default" loading={importingId === b.campaign_id} onClick={() => handleImportVendor(b.campaign_id!)}>📥 </Button>
</>
)}
{b.stage === "returned" && <span className="text-green-600 text-sm font-medium"> </span>}
{b.stage === "returned" && (
<>
{b.project === "adas" && b.campaign_id && (
<Button size="small" variant="default" loading={fittingId === b.campaign_id} onClick={() => handleCuboidFit(b.campaign_id!)}>
3D
</Button>
)}
<Button
size="small"
variant="primary"
loading={buildingId === (b.campaign_id || b.batch)}
onClick={() => handleSubmitBuild(b)}
>
🏗 build
</Button>
<Link to="/system/audit" className="text-xs text-blue-600 hover:underline"> </Link>
</>
)}
</div>
</div>
</div>
@@ -147,8 +218,8 @@ export const ExportPage: React.FC = () => {
</div>
) : (
<div className="card text-center py-12">
<p className="text-gray-400 text-lg mb-3"></p>
<p className="text-gray-400 text-sm mb-4"></p>
<p className="text-gray-400 text-lg mb-3"> build </p>
<p className="text-gray-400 text-sm mb-4"></p>
<Link to="/labeling/campaigns"><Button variant="default" size="small"> </Button></Link>
</div>
)}

View File

@@ -33,7 +33,7 @@ const ReviewListPage: React.FC = () => {
const results: LabelingBatchRow[] = [];
const [inReview, approved, rejected] = await Promise.allSettled([
hsapApi.labelingBatches({ stage: "in_review", limit: 100 }),
hsapApi.labelingBatches({ stage: "review_approved", limit: 50 }),
hsapApi.labelingBatches({ stage: "labeling_submitted", limit: 50 }),
hsapApi.labelingBatches({ stage: "review_rejected", limit: 50 }),
]);
if (inReview.status === "fulfilled") results.push(...((inReview.value.items || []) as LabelingBatchRow[]));
@@ -80,7 +80,7 @@ const ReviewListPage: React.FC = () => {
{/* Filter chips */}
<div className="flex gap-1.5">
{["全部", "质检中", "已通过", "已退回"].map((label, i) => {
const val = i === 0 ? "" : ["in_review", "review_approved", "review_rejected"][i - 1];
const val = i === 0 ? "" : ["in_review", "labeling_submitted", "review_rejected"][i - 1];
return (
<button key={val} onClick={() => setStageFilter(val)}
className={`px-3 py-1.5 rounded-lg text-xs font-medium transition-colors ${
@@ -120,7 +120,9 @@ const ReviewListPage: React.FC = () => {
</div>
<div className="shrink-0">
{b.stage === "in_review" && <Link to={`/labeling/review/${b.campaign_id}`}><Button size="small" variant="primary"> </Button></Link>}
{b.stage === "review_approved" && <span className="text-green-600 text-sm font-medium"> </span>}
{b.stage === "labeling_submitted" && (
<Link to="/labeling/export"><Button size="small" variant="default"> </Button></Link>
)}
{b.stage === "review_rejected" && <span className="text-red-600 text-sm font-medium"> 退</span>}
</div>
</div>

View File

@@ -198,6 +198,12 @@ export const WorkbenchPage: React.FC = () => {
</Link>
)}
{b.stage === "returned" && (
<Link to="/labeling/export"
className="inline-flex items-center gap-1 px-3 py-1.5 text-xs font-medium rounded-lg bg-orange-50 text-orange-700 hover:bg-orange-100 transition-colors">
🏗 build
</Link>
)}
</div>
</div>
</div>