feat: 合并 Docker Compose、标注表格优化与部署文档

将 platform + CVAT 合并为单文件 docker-compose.yml,完善 .env 与 init/dev_up 脚本;
新增 docs/DEPLOY.md 与更新 README 以支持新机器部署;含数据湖示例、车队地图、
紧凑表格 UI、ADAS det_7cls 路径与批次台账等近期改动。

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
2026-06-16 17:06:31 +08:00
parent 0b8ade048e
commit 483e027482
117 changed files with 5933 additions and 1499 deletions

View File

@@ -45,6 +45,45 @@ class DeliveryPatchBody(BaseModel):
owner_name: str | None = None
@router.get("/scan")
def api_scan_deliveries(
_user: Annotated[User, Depends(require_any_permission("read:deliveries", "read:pending", "*"))],
projects: str | None = Query(None, description="逗号分隔: dms,adas,lane"),
) -> dict[str, Any]:
from as_platform.deliveries.scan import scan_delivery_sources
projs = [p.strip() for p in projects.split(",") if p.strip()] if projects else None
return scan_delivery_sources(projects=projs)
class ScanRegisterBody(BaseModel):
items: list[dict[str, Any]] = Field(default_factory=list)
sync_workbench: bool = True
@router.post("/scan/register")
def api_register_scanned_deliveries(
body: ScanRegisterBody,
user: Annotated[User, Depends(require_any_permission("write:delivery_submit", "*"))],
) -> dict[str, Any]:
from as_platform.deliveries.scan import register_scanned_to_ledger
return register_scanned_to_ledger(body.items, user, sync_workbench=body.sync_workbench)
@router.post("/{delivery_id}/sync-workbench")
def api_sync_delivery_workbench(
delivery_id: str,
_user: Annotated[User, Depends(require_any_permission("write:delivery_submit", "*"))],
) -> dict[str, Any]:
from as_platform.deliveries.scan import bridge_delivery_to_workbench
try:
return bridge_delivery_to_workbench(delivery_id)
except ValueError as e:
raise HTTPException(400, str(e)) from e
@router.get("")
def api_list_deliveries(
_user: Annotated[User, Depends(require_any_permission("read:deliveries", "read:pending", "*"))],

View File

@@ -128,10 +128,40 @@ def api_assign_campaign(
def api_labeling_batches(
_user: Annotated[User, Depends(require_permission("read:pending"))],
stage: str | None = Query(None),
stages: str | None = Query(None, description="逗号分隔多阶段,一次扫描返回"),
offset: int = Query(0, ge=0),
limit: int = Query(20, ge=1, le=100),
refresh: bool = Query(False, description="true 时先重建索引再返回"),
q: str | None = Query(None, description="搜索批次名/任务/项目"),
) -> dict[str, Any]:
return list_labeling_batches(stage=stage, offset=offset, limit=limit)
stage_list = [s.strip() for s in stages.split(",")] if stages else None
return list_labeling_batches(
stage=stage, stages=stage_list, offset=offset, limit=limit, refresh=refresh, q=q,
)
@router.post("/api/v1/labeling/batches/rebuild-index")
def api_rebuild_batch_index(
_user: Annotated[User, Depends(require_permission("write:labeling_assign"))],
) -> dict[str, Any]:
from as_platform.labeling.batch_index import rebuild_batch_index
return rebuild_batch_index()
@router.post("/api/v1/labeling/batches/{campaign_id}/archive")
def api_archive_batch(
campaign_id: str,
_user: Annotated[User, Depends(require_permission("write:labeling_assign"))],
) -> dict[str, Any]:
from as_platform.labeling.batch_index import archive_batch
try:
return archive_batch(campaign_id)
except FileNotFoundError:
raise HTTPException(404, "batch not found") from None
except ValueError as e:
raise HTTPException(400, str(e)) from e
@router.post("/api/v1/labeling/campaigns/open")
@@ -488,23 +518,7 @@ def api_review_image(
from as_platform.audit.review import get_review_image
import tempfile
try:
# Get class names from registry
import yaml
from as_platform.data.core import load_wf, proj_root
wf = load_wf()
root = proj_root(wf, "dms")
reg = yaml.safe_load((root / wf["projects"]["dms"]["registry"]).read_text())
# Build class_names dict from campaign scope
class_names: dict[int, str] = {}
# Try to get from the specific task
tasks = reg.get("tasks", {})
for task_cfg in tasks.values():
names = task_cfg.get("names")
if isinstance(names, list):
for i, n in enumerate(names):
class_names[i] = n
data = get_review_image(campaign_id, path, class_names)
data = get_review_image(campaign_id, path)
tmp = tempfile.NamedTemporaryFile(suffix=".jpg", delete=False)
tmp.write(data)
tmp.close()
@@ -528,6 +542,16 @@ def api_review_submit(
def api_review_progress(
campaign_id: str,
_user: Annotated[User, Depends(require_permission("read:pending"))],
) -> dict[str, int]:
) -> dict[str, Any]:
from as_platform.audit.review import review_progress
return review_progress(campaign_id)
@router.get("/api/v1/labeling/review-progress")
def api_review_progress_batch(
_user: Annotated[User, Depends(require_permission("read:pending"))],
campaign_ids: str = Query(..., description="逗号分隔 campaign id最多 50 个"),
) -> dict[str, Any]:
from as_platform.audit.review import review_progress_batch
ids = [x.strip() for x in campaign_ids.split(",") if x.strip()]
return review_progress_batch(ids)

View File

@@ -693,7 +693,8 @@ def api_scan_inbox(
project: str = Query("dms"),
) -> dict[str, Any]:
"""扫描 inbox 目录,返回未登记的新批次。"""
from as_platform.data.core import get_pending_report, load_wf, proj_root
from as_platform.data.core import load_wf, proj_root
from as_platform.labeling.batch_index import index_is_empty, rebuild_batch_index
wf = load_wf()
root = proj_root(wf, project)
@@ -701,8 +702,20 @@ def api_scan_inbox(
if not inbox.is_dir():
return {"project": project, "items": [], "inbox_path": str(inbox)}
report = get_pending_report()
registered = {b.get("batch", "") for b in report.get("batches", [])}
if index_is_empty():
rebuild_batch_index(wf)
from as_platform.db.engine import session_scope
from as_platform.db.models import BatchIndex
with session_scope() as db:
registered = {
(r.task or "", r.batch)
for r in db.query(BatchIndex).filter(
BatchIndex.project == project,
BatchIndex.archived.is_(False),
).all()
}
items: list[dict[str, Any]] = []
for task_dir in sorted(inbox.iterdir()):
@@ -713,7 +726,7 @@ def api_scan_inbox(
continue
batch_name = batch_dir.name
task_name = task_dir.name
if batch_name in registered:
if (task_name, batch_name) in registered:
continue # 已登记
# Count images (含 images/ 子目录)

View File

@@ -58,11 +58,107 @@ def _parse_labels(label_path: Path) -> list[dict[str, Any]]:
results = []
for line in label_path.read_text().strip().splitlines():
ann = _parse_yolo_line(line)
if ann:
if ann and ann["bbox"][2] > 0 and ann["bbox"][3] > 0:
results.append(ann)
return results
def _class_names_for_campaign(camp) -> dict[int, str]:
"""campaign task → class_id → name。"""
import yaml
from as_platform.data.core import load_wf, proj_root
if not camp or camp.project != "dms":
return {}
wf = load_wf()
root = proj_root(wf, "dms")
reg = yaml.safe_load((root / wf["projects"]["dms"]["registry"]).read_text(encoding="utf-8")) or {}
tcfg = (reg.get("tasks") or {}).get(camp.task) or {}
if camp.mode and tcfg.get("type") == "multi":
mcfg = (tcfg.get("modes") or {}).get(camp.mode) or {}
names = mcfg.get("names")
else:
names = tcfg.get("names")
if isinstance(names, list):
return {i: str(n) for i, n in enumerate(names)}
if isinstance(names, dict):
return {int(k): str(v) for k, v in names.items()}
return {}
def _name_to_class_id(name: str, class_names: dict[int, str]) -> int:
rev = {v.lower(): k for k, v in class_names.items()}
return rev.get(name.lower(), 0)
def _resolve_yolo_label_path(batch_dir: Path, img_path: Path) -> Path | None:
stem = img_path.stem
for rel in (
f"labels/{stem}.txt",
f"labels/train/{stem}.txt",
f"labels/val/{stem}.txt",
f"labels/yolo/{stem}.txt",
):
p = batch_dir / rel
if p.is_file():
return p
return None
def _parse_ls_annotations(path: Path, class_names: dict[int, str]) -> list[dict[str, Any]]:
import json
try:
data = json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
return []
out: list[dict[str, Any]] = []
for item in data.get("result") or []:
if item.get("type") not in ("rectanglelabels", "rectangle"):
continue
val = item.get("value") or {}
w_pct = float(val.get("width") or 0)
h_pct = float(val.get("height") or 0)
if w_pct <= 0 or h_pct <= 0:
continue
x_pct = float(val.get("x") or 0)
y_pct = float(val.get("y") or 0)
labels = val.get("rectanglelabels") or val.get("labels") or []
label = labels[0] if labels else "unknown"
cid = _name_to_class_id(str(label), class_names)
cx = (x_pct + w_pct / 2) / 100.0
cy = (y_pct + h_pct / 2) / 100.0
out.append({"class_id": cid, "bbox": (cx, cy, w_pct / 100.0, h_pct / 100.0)})
return out
def _load_image_annotations(
batch_dir: Path,
img_path: Path,
class_names: dict[int, str],
) -> list[dict[str, Any]]:
yolo = _resolve_yolo_label_path(batch_dir, img_path)
if yolo:
anns = _parse_labels(yolo)
if anns:
return anns
from as_platform.labeling.annotate import _task_id_for_image
ann_json = batch_dir / "labels" / "ls_annotations" / f"{_task_id_for_image(img_path, batch_dir)}.json"
if ann_json.is_file():
return _parse_ls_annotations(ann_json, class_names)
return []
def _image_has_labels(batch_dir: Path, img_path: Path, class_names: dict[int, str]) -> bool:
return bool(_load_image_annotations(batch_dir, img_path, class_names))
def _list_review_images(batch_dir: Path) -> list[Path]:
from as_platform.labeling.annotate import _iter_batch_images
return list(_iter_batch_images(batch_dir))
# ── Optimized overlay render ──
PALETTE = [(220, 20, 60), (30, 144, 255), (50, 205, 50), (255, 165, 0), (186, 85, 211), (0, 206, 209)]
@@ -70,7 +166,7 @@ PALETTE = [(220, 20, 60), (30, 144, 255), (50, 205, 50), (255, 165, 0), (186, 85
def render_review_overlay(
image_path: Path,
label_path: Path | None,
batch_dir: Path,
class_names: dict[int, str],
*,
max_size: int = 800,
@@ -88,7 +184,7 @@ def render_review_overlay(
font = _get_font(max(12, min(16, w // 50)))
line_w = max(1, w // 400)
anns = _parse_labels(label_path) if label_path else []
anns = _load_image_annotations(batch_dir, image_path, class_names)
for ann in anns:
cid = ann["class_id"]
color = PALETTE[cid % len(PALETTE)]
@@ -140,17 +236,14 @@ def get_review_queue(campaign_id: str, offset: int = 0, limit: int = 20) -> dict
if not camp:
return {"items": [], "total": 0, "hint": "Campaign 不存在"}
batch_dir = resolve_campaign_batch_dir(camp)
class_names = _class_names_for_campaign(camp)
if not batch_dir or not batch_dir.is_dir():
return {"items": [], "total": 0, "hint": "批次目录不存在"}
img_dir = batch_dir / "images"
if not img_dir.is_dir():
all_images = _list_review_images(batch_dir)
if not all_images:
return {"items": [], "total": 0, "hint": "无 images 目录"}
all_images: list[Path] = []
for ext in IMAGE_EXTS:
all_images.extend(sorted(img_dir.rglob(f"*{ext}")))
# Get existing reviews
with session_scope() as db:
reviewed = {
@@ -160,26 +253,26 @@ def get_review_queue(campaign_id: str, offset: int = 0, limit: int = 20) -> dict
total = len(all_images)
page = all_images[offset:offset + limit]
score_counts = {"good": 0, "fine": 0, "bad": 0, "pending": 0}
items = []
for img in page:
rel = str(img.relative_to(batch_dir))
score = reviewed.get(rel, "pending")
score_counts[score] += 1
label_path = batch_dir / "labels" / (img.stem + ".txt")
items.append({
"id": rel, "image_path": rel,
"fileName": img.name,
"score": score,
"has_label": label_path.is_file(),
"has_label": _image_has_labels(batch_dir, img, class_names),
})
# Fill remaining counts
for img in all_images:
rel = str(img.relative_to(batch_dir))
s = reviewed.get(rel, "pending")
if s not in score_counts:
score_counts[s] = 0
with session_scope() as db:
db_counts = _review_db_counts(db, campaign_id)
reviewed_n = sum(db_counts.values())
score_counts = {
"good": db_counts.get("good", 0),
"fine": db_counts.get("fine", 0),
"bad": db_counts.get("bad", 0),
"pending": max(0, total - reviewed_n),
}
return {
"items": items, "total": total,
@@ -188,7 +281,7 @@ def get_review_queue(campaign_id: str, offset: int = 0, limit: int = 20) -> dict
}
def get_review_image(campaign_id: str, image_rel_path: str, class_names: dict[int, str]) -> bytes:
def get_review_image(campaign_id: str, image_rel_path: str) -> bytes:
from as_platform.labeling.annotate import resolve_campaign_batch_dir
from as_platform.db.engine import session_scope
from as_platform.db.models import LabelingCampaign
@@ -197,11 +290,13 @@ def get_review_image(campaign_id: str, image_rel_path: str, class_names: dict[in
if not camp:
raise FileNotFoundError("Campaign 不存在")
batch_dir = resolve_campaign_batch_dir(camp)
class_names = _class_names_for_campaign(camp)
if not batch_dir:
raise FileNotFoundError("批次不存在")
img_path = batch_dir / image_rel_path
lbl_path = batch_dir / "labels" / (img_path.stem + ".txt")
return render_review_overlay(img_path, lbl_path if lbl_path.is_file() else None, class_names)
if not img_path.is_file():
raise FileNotFoundError(f"图片不存在: {image_rel_path}")
return render_review_overlay(img_path, batch_dir, class_names)
def submit_review_scores(
@@ -251,22 +346,76 @@ def submit_review_scores(
reviewed = sum(counts.values())
if reviewed >= total_images and total_images > 0:
pass_rate = counts.get("good", 0) / max(total_images, 1)
new_stage = "review_approved" if pass_rate >= 0.8 else "review_rejected"
_update_campaign_stage(db, campaign_id, new_stage)
new_stage = _effective_stage_from_review(
counts.get("good", 0), counts.get("fine", 0), counts.get("bad", 0), total_images,
)
if new_stage and new_stage != "in_review":
raw = "review_approved" if new_stage == "labeling_submitted" else new_stage
_update_campaign_stage(db, campaign_id, raw)
return {"ok": True, "updated": updated, "auto_advanced": reviewed >= total_images if total_images > 0 else False}
auto_advanced = reviewed >= total_images if total_images > 0 else False
acceptable = counts.get("good", 0) + counts.get("fine", 0) if total_images > 0 else 0
final_stage = None
if auto_advanced and total_images > 0:
eff = _effective_stage_from_review(
counts.get("good", 0), counts.get("fine", 0), counts.get("bad", 0), total_images,
)
final_stage = "review_approved" if eff == "labeling_submitted" else eff
return {
"ok": True,
"updated": updated,
"auto_advanced": auto_advanced,
"stage": final_stage,
}
def _review_db_counts(db, campaign_id: str) -> dict[str, int]:
from sqlalchemy import func
from collections import Counter
rows = db.query(LabelingReview.score, func.count()).filter(
LabelingReview.campaign_id == campaign_id
).group_by(LabelingReview.score).all()
return {score: cnt for score, cnt in rows}
PASS_RATE_THRESHOLD = 0.8
def _effective_stage_from_review(good: int, fine: int, bad: int, total: int) -> str | None:
"""Return campaign status after QA is complete; None if images remain unreviewed."""
if total <= 0:
return None
reviewed = good + fine + bad
if reviewed < total:
return "in_review"
acceptable = good + fine
approved = acceptable / total >= PASS_RATE_THRESHOLD
return "labeling_submitted" if approved else "review_rejected"
def reconcile_review_stage(campaign_id: str) -> str | None:
"""Align stored campaign stage with current review scores (fixes stale rejections)."""
summary = _review_summary(campaign_id)
if not summary.get("complete"):
return summary.get("stage")
expected = _effective_stage_from_review(
summary["good"], summary["fine"], summary["bad"], summary["total"],
)
if not expected:
return summary.get("stage")
with session_scope() as db:
from as_platform.db.models import LabelingCampaign
camp = db.get(LabelingCampaign, campaign_id)
if not camp:
return None
if camp.status == expected:
return expected
camp.status = expected
from as_platform.labeling.batch_stage import update_campaign_batch_meta_stage
update_campaign_batch_meta_stage(camp, expected)
db.commit()
return expected
def _update_campaign_stage(db, campaign_id: str, new_stage: str) -> None:
from as_platform.db.models import LabelingCampaign
from as_platform.labeling.batch_stage import update_campaign_batch_meta_stage
@@ -278,10 +427,63 @@ def _update_campaign_stage(db, campaign_id: str, new_stage: str) -> None:
update_campaign_batch_meta_stage(camp, effective)
def review_progress(campaign_id: str) -> dict[str, int]:
def _review_summary(campaign_id: str) -> dict[str, Any]:
from as_platform.labeling.annotate import resolve_campaign_batch_dir
from as_platform.db.models import LabelingCampaign
with session_scope() as db:
rows = db.query(LabelingReview).filter(LabelingReview.campaign_id == campaign_id).all()
counts = {"good": 0, "fine": 0, "bad": 0, "pending": 0}
for r in rows:
counts[r.score] = counts.get(r.score, 0) + 1
return counts
camp = db.get(LabelingCampaign, campaign_id)
if not camp:
return {"good": 0, "fine": 0, "bad": 0, "pending": 0, "total": 0, "reviewed": 0, "pass_rate": 0, "complete": False, "stage": ""}
batch_dir = resolve_campaign_batch_dir(camp)
stage = camp.status or ""
if not batch_dir or not batch_dir.is_dir():
counts = _review_db_counts(db, campaign_id)
reviewed = sum(counts.values())
return {
**{k: counts.get(k, 0) for k in ("good", "fine", "bad")},
"pending": 0,
"total": reviewed,
"reviewed": reviewed,
"pass_rate": round((counts.get("good", 0) + counts.get("fine", 0)) / max(reviewed, 1) * 100),
"complete": reviewed > 0,
"stage": stage,
}
all_images = _list_review_images(batch_dir)
db_counts = _review_db_counts(db, campaign_id)
total = len(all_images)
good = db_counts.get("good", 0)
fine = db_counts.get("fine", 0)
bad = db_counts.get("bad", 0)
reviewed = good + fine + bad
acceptable = good + fine
return {
"good": good,
"fine": fine,
"bad": bad,
"pending": max(0, total - reviewed),
"total": total,
"reviewed": reviewed,
"pass_rate": round(acceptable / max(total, 1) * 100),
"complete": reviewed >= total and total > 0,
"stage": stage,
}
def review_progress(campaign_id: str) -> dict[str, Any]:
result = _review_summary(campaign_id)
if result.get("complete"):
reconciled = reconcile_review_stage(campaign_id)
if reconciled:
result["stage"] = reconciled
return result
def review_progress_batch(campaign_ids: list[str]) -> dict[str, Any]:
ids = [c.strip() for c in campaign_ids if c and c.strip()][:50]
items: dict[str, Any] = {}
for cid in ids:
items[cid] = review_progress(cid)
return {"items": items}

View File

@@ -1098,15 +1098,22 @@ def register_batch(
meta_path = write_meta(batch_dir, data)
invalidate_catalog_cache()
batch_row = enrich_batch(
batch_dir,
project=project,
task=task,
pack=pack,
batch=batch,
location=location,
)
try:
from as_platform.labeling.batch_index import upsert_batch_dict
upsert_batch_dict(batch_row)
except Exception:
pass
return {
"ok": True,
"meta_path": str(meta_path),
"batch": enrich_batch(
batch_dir,
project=project,
task=task,
pack=pack,
batch=batch,
location=location,
),
"batch": batch_row,
}

View File

@@ -425,6 +425,11 @@ def promote_candidate_to_inbox(
raise ValueError(f"任务 {task} 为 multi须指定 mode如 batch_0516")
dest = _resolve_dms_inbox_dest(root, reg, task, eff_mode, batch_name)
reg_batch = eff_mode or batch_name
elif project == "adas":
if not task:
raise ValueError("ADAS 晋级需要 taskdet_7cls 或 cuboid_7cls")
dest = root / "inbox" / task / batch_name
reg_batch = batch_name
else:
dest = root / "inbox" / batch_name
reg_batch = batch_name
@@ -439,7 +444,7 @@ def promote_candidate_to_inbox(
row = enrich_batch(
dest,
project=project,
task=task if project == "dms" else None,
task=task,
pack=None,
batch=reg_batch,
location="inbox",
@@ -482,6 +487,12 @@ def promote_candidate_to_inbox(
db.flush()
invalidate_catalog_cache()
try:
from as_platform.labeling.batch_index import upsert_batch_dict
upsert_batch_dict({**meta, "path": str(dest), "location": "inbox"})
except Exception:
pass
return {
"ok": True,
"candidate_id": candidate_id,

View File

@@ -6,20 +6,35 @@ from pathlib import Path
from as_platform.data.promote.base import PackPromoteAdapter, PromoteContext, PromoteResult
from as_platform.data.promote.manifest import refresh_dms_yaml
from as_platform.data.promote.validate.dms_yolo import validate_dms_task
from as_platform.data.promote.validate.dms_yolo import validate_dms_inbox_batch
_DMS_SCRIPTS = Path(__file__).resolve().parents[4] / "datasets" / "dms" / "scripts"
if str(_DMS_SCRIPTS) not in sys.path:
sys.path.insert(0, str(_DMS_SCRIPTS))
def _resolve_promote_pack_dir(project_root: Path, pack: str) -> Path:
"""解析 pack 目录;损坏的 workspace 软链则回退为 HSAP 内真实目录。"""
candidate = project_root / "packs" / pack
if candidate.is_symlink():
try:
resolved = candidate.resolve()
if resolved.is_dir():
return resolved
except OSError:
pass
candidate.unlink()
candidate.mkdir(parents=True, exist_ok=True)
return candidate
class DmsYoloPromoteAdapter(PackPromoteAdapter):
project = "dms"
def validate(self, ctx: PromoteContext) -> list[str]:
if ctx.skip_validate:
return []
return validate_dms_task(ctx.task)
return validate_dms_inbox_batch(ctx.batch_dir)
def promote(self, ctx: PromoteContext) -> PromoteResult:
from ingest_incremental import promote_inbox_batch
@@ -34,8 +49,7 @@ class DmsYoloPromoteAdapter(PackPromoteAdapter):
warnings=[f"batch_dir missing: {ctx.batch_dir}"],
)
pack_dir = ctx.project_root / "packs" / ctx.pack
pack_dir.mkdir(parents=True, exist_ok=True)
pack_dir = _resolve_promote_pack_dir(ctx.project_root, ctx.pack)
detail = promote_inbox_batch(
root=ctx.project_root,
@@ -46,7 +60,7 @@ class DmsYoloPromoteAdapter(PackPromoteAdapter):
dry_run=ctx.dry_run,
refresh=ctx.refresh and not ctx.dry_run,
)
if ctx.refresh and not ctx.dry_run and not ctx.skip_validate:
if ctx.refresh and not ctx.dry_run:
refresh_dms_yaml(task=ctx.task)
added = int(detail.get("added") or 0)

View File

@@ -8,6 +8,17 @@ from pathlib import Path
from as_platform.config import WORKSPACE
def validate_dms_inbox_batch(batch_dir: Path) -> list[str]:
"""Promote 前校验单个 inbox 批次(不要求 pack 目录已存在)。"""
from as_platform.labeling.batch_stage import batch_has_yolo_labels
if not batch_dir.is_dir():
return [f"batch_dir missing: {batch_dir}"]
if not batch_has_yolo_labels(batch_dir):
return [f"no YOLO labels under {batch_dir} (先执行 labeling_export)"]
return []
def validate_dms_task(task: str | None) -> list[str]:
cmd = [sys.executable, str(WORKSPACE / "scripts" / "validate_dms_tasks.py")]
if task:

View File

@@ -28,7 +28,7 @@ ROLE_DEFS: dict[str, tuple[str, list[str]]] = {
]),
"engineer": ("算法工程师", [
"read:catalog", "read:pending", "read:jobs", "read:audit", "read:fleet", "write:fleet",
"write:approval_submit", "write:delivery_submit", "read:deliveries",
"write:approval_submit", "write:approval_review", "write:delivery_submit", "read:deliveries",
"write:labeling_vendor", "write:labeling_assign",
]),
"labeler": ("标注协调", [
@@ -79,6 +79,7 @@ def init_database() -> None:
_ensure_feishu_bitable_columns(db)
_ensure_approval_columns(db)
_ensure_operation_log_columns(db)
_ensure_batch_index_columns(db)
_seed_roles_permissions(db)
_seed_fleet_demo(db)
_import_jsonl_if_empty(db)
@@ -168,18 +169,22 @@ def _import_jsonl_if_empty(db: Session) -> None:
def assign_default_role(db: Session, user: User) -> None:
"""新用户默认角色;支持 open_id / 部门白名单自动 admin。"""
"""新用户默认角色;白名单用户每次登录也会补齐 admin。"""
admin_role = db.query(Role).filter_by(code="admin").first()
user_dept_ids = set(user.feishu_department_ids())
if user.feishu_open_id and user.feishu_open_id in FEISHU_ADMIN_OPEN_IDS:
role_code = "admin"
elif user_dept_ids and user_dept_ids.intersection(FEISHU_ADMIN_DEPARTMENT_IDS):
role_code = "admin"
elif not user.roles:
role_code = "engineer"
else:
if admin_role and admin_role not in user.roles:
user.roles.append(admin_role)
return
if user_dept_ids and user_dept_ids.intersection(FEISHU_ADMIN_DEPARTMENT_IDS):
if admin_role and admin_role not in user.roles:
user.roles.append(admin_role)
return
if user.roles:
return
role_code = "engineer"
role = db.query(Role).filter_by(code=role_code).first()
if role and role not in user.roles:
if role:
user.roles.append(role)
@@ -325,6 +330,10 @@ def _ensure_approval_columns(db: Session) -> None:
_ensure_table_columns(db, "approvals", {"rejection_category": "VARCHAR(32) DEFAULT ''"})
def _ensure_batch_index_columns(db: Session) -> None:
_ensure_table_columns(db, "batch_index", {"archived": "BOOLEAN DEFAULT FALSE"})
def _ensure_operation_log_columns(db: Session) -> None:
"""确保 operation_logs 表存在并包含所有列。"""
inspector_args = {}

View File

@@ -319,6 +319,67 @@ class BatchDelivery(Base):
}
class BatchIndex(Base):
"""批次列表索引:由扫盘/登记/阶段变更写入,列表 API 只读此表。"""
__tablename__ = "batch_index"
campaign_id = Column(String(64), primary_key=True)
project = Column(String(32), nullable=False, index=True)
task = Column(String(64), nullable=True, index=True)
mode = Column(String(64), nullable=True)
batch = Column(String(128), nullable=False, index=True)
pack = Column(String(64), nullable=True)
location = Column(String(32), nullable=False, default="inbox")
stage = Column(String(32), nullable=False, index=True)
batch_path = Column(String(1024), nullable=True)
scope_key = Column(String(128), nullable=True)
engineer = Column(String(128), nullable=True)
format = Column(String(32), nullable=True)
image_count = Column(Integer, nullable=False, default=0)
label_count = Column(Integer, nullable=False, default=0)
has_meta = Column(Boolean, nullable=False, default=False)
registry_only = Column(Boolean, nullable=False, default=False)
next_cli = Column(String(512), nullable=True)
domain = Column(String(32), nullable=True)
domain_label = Column(String(64), nullable=True)
task_label = Column(String(128), nullable=True)
mode_label = Column(String(128), nullable=True)
labeling_profile = Column(String(128), nullable=True)
export_default = Column(String(128), nullable=True)
ml_adapter = Column(String(128), nullable=True)
indexed_at = Column(DateTime(timezone=True), nullable=False)
archived = Column(Boolean, nullable=False, default=False, index=True)
def to_list_row(self) -> dict:
return {
"project": self.project,
"task": self.task,
"mode": self.mode,
"batch": self.batch,
"pack": self.pack,
"stage": self.stage,
"location": self.location,
"path": self.batch_path,
"engineer": self.engineer,
"format": self.format,
"counts": {"images": self.image_count, "labels": self.label_count},
"has_meta": self.has_meta,
"next_cli": self.next_cli,
"scope_key": self.scope_key,
"domain": self.domain,
"domain_label": self.domain_label,
"task_label": self.task_label,
"mode_label": self.mode_label,
"labeling_profile": self.labeling_profile,
"export_default": self.export_default,
"ml_adapter": self.ml_adapter,
"campaign_id": self.campaign_id,
"registry_only": self.registry_only,
"indexed_at": self.indexed_at.isoformat() if self.indexed_at else None,
}
class FeishuBitableLink(Base):
"""HSAP 批次与飞书多维表格行的对应关系。"""

View File

@@ -0,0 +1,230 @@
"""扫描 inbox / 数据湖目录,与批次台账对齐。"""
from __future__ import annotations
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from as_platform.data.core import load_wf, proj_root, register_batch
from as_platform.db.engine import session_scope
from as_platform.db.models import BatchDelivery, BatchIndex, User
from as_platform.deliveries.service import _new_delivery_id, _normalize_task
def _utcnow() -> datetime:
return datetime.now(timezone.utc)
def _dir_mtime_iso(path: Path) -> str | None:
try:
ts = path.stat().st_mtime
return datetime.fromtimestamp(ts, tz=timezone.utc).strftime("%Y-%m-%d")
except OSError:
return None
def _scan_project_inbox(project: str, wf: dict | None = None) -> list[dict[str, Any]]:
from as_platform.data.batch import count_images, count_label_files, dms_has_labels
wf = wf or load_wf()
root = proj_root(wf, project)
inbox = root / "inbox"
if not inbox.is_dir():
return []
with session_scope() as db:
deliveries = {
(r.project, r.task or "", r.mode or "", r.batch_name): r
for r in db.query(BatchDelivery).filter(BatchDelivery.project == project).all()
}
indexed = {
(r.task or "", r.batch)
for r in db.query(BatchIndex).filter(
BatchIndex.project == project,
BatchIndex.archived.is_(False),
).all()
}
items: list[dict[str, Any]] = []
for task_dir in sorted(inbox.iterdir()):
if not task_dir.is_dir():
continue
for batch_dir in sorted(task_dir.iterdir()):
if not batch_dir.is_dir():
continue
task_name = task_dir.name
batch_name = batch_dir.name
img_count = count_images(batch_dir)
if not img_count and (batch_dir / "images").is_dir():
img_count = count_images(batch_dir / "images")
lbl_count = count_label_files(batch_dir / "labels") if (batch_dir / "labels").is_dir() else 0
has_labels = lbl_count > 0 or dms_has_labels(batch_dir)
stage_hint = "returned" if has_labels and lbl_count > 0 else "raw_pool"
key = (project, task_name, "", batch_name)
delivery = deliveries.get(key)
in_index = (task_name, batch_name) in indexed
items.append({
"project": project,
"task": task_name,
"mode": None,
"batch": batch_name,
"batch_name": batch_name,
"path": str(batch_dir),
"data_path": str(batch_dir),
"images": img_count,
"labels": lbl_count,
"has_labels": has_labels,
"stage_hint": stage_hint,
"source_type": "inbox_scan",
"delivery_id": delivery.id if delivery else None,
"delivery_status": delivery.status if delivery else None,
"in_ledger": delivery is not None,
"in_workbench": in_index,
"collection_start": delivery.collection_start if delivery else _dir_mtime_iso(batch_dir),
"collection_end": delivery.collection_end if delivery else None,
"created_at": delivery.created_at.isoformat() if delivery and delivery.created_at else None,
"needs_ledger": delivery is None,
"needs_workbench": not in_index,
})
return items
def scan_delivery_sources(*, projects: list[str] | None = None) -> dict[str, Any]:
"""扫描 inbox返回与台账、工作台对齐状态。"""
projs = projects or ["dms", "adas", "lane"]
wf = load_wf()
items: list[dict[str, Any]] = []
for p in projs:
items.extend(_scan_project_inbox(p, wf))
needs_ledger = sum(1 for i in items if i.get("needs_ledger"))
needs_workbench = sum(1 for i in items if i.get("needs_workbench"))
return {
"items": items,
"count": len(items),
"needs_ledger": needs_ledger,
"needs_workbench": needs_workbench,
"scanned_at": _utcnow().isoformat(),
}
def register_scanned_to_ledger(
items: list[dict[str, Any]],
user: User,
*,
sync_workbench: bool = True,
) -> dict[str, Any]:
"""将扫描结果登记到台账;已在 inbox 的批次直接标为 in_lake 并同步工作台。"""
created = 0
updated = 0
synced = 0
out_items: list[dict[str, Any]] = []
for raw in items:
project = (raw.get("project") or "dms").strip()
task = _normalize_task(project, raw.get("task"))
mode = (raw.get("mode") or "").strip() or None
batch_name = (raw.get("batch_name") or raw.get("batch") or "").strip()
data_path = (raw.get("data_path") or raw.get("path") or "").strip()
if not batch_name or not data_path:
continue
if not Path(data_path).is_dir():
continue
stage_hint = raw.get("stage_hint") or "raw_pool"
collection_start = (raw.get("collection_start") or "").strip() or _dir_mtime_iso(Path(data_path))
collection_end = (raw.get("collection_end") or "").strip() or None
estimated = raw.get("images")
if estimated is None:
estimated = raw.get("estimated_count")
with session_scope() as db:
rec = (
db.query(BatchDelivery)
.filter_by(project=project, task=task, mode=mode, batch_name=batch_name)
.first()
)
if not rec:
rec = BatchDelivery(
id=_new_delivery_id(),
project=project,
task=task,
mode=mode,
batch_name=batch_name,
source_type=(raw.get("source_type") or "inbox_scan"),
collection_start=collection_start,
collection_end=collection_end,
data_path=data_path,
estimated_count=int(estimated) if estimated not in (None, "") else None,
status="in_lake",
inbox_path=data_path,
owner_user_id=user.id,
owner_name=user.name,
submitted_by_user_id=user.id,
submitted_by_name=user.name,
)
db.add(rec)
created += 1
else:
if rec.status in ("draft", "rejected", "ingest_failed"):
rec.status = "in_lake"
if not rec.inbox_path:
rec.inbox_path = data_path
if not rec.data_path:
rec.data_path = data_path
if collection_start and not rec.collection_start:
rec.collection_start = collection_start
if estimated not in (None, "") and not rec.estimated_count:
rec.estimated_count = int(estimated)
if not rec.source_type:
rec.source_type = "inbox_scan"
rec.updated_at = _utcnow()
updated += 1
db.flush()
out_items.append(rec.to_dict())
if sync_workbench and stage_hint in ("raw_pool", "returned"):
try:
register_batch(
None,
project,
task,
batch_name,
stage=stage_hint,
location="inbox",
)
synced += 1
except Exception:
pass
return {
"ok": True,
"created": created,
"updated": updated,
"synced_workbench": synced,
"items": out_items,
}
def bridge_delivery_to_workbench(delivery_id: str) -> dict[str, Any]:
"""台账 in_lake 后同步到送标工作台索引。"""
with session_scope() as db:
rec = db.get(BatchDelivery, delivery_id)
if not rec:
raise ValueError("送标申请不存在")
if rec.status != "in_lake":
raise ValueError(f"当前状态不可同步工作台: {rec.status}")
project = rec.project
task = rec.task
batch_name = rec.batch_name
inbox_path = rec.inbox_path or rec.data_path
stage = "raw_pool"
if inbox_path:
labels_dir = Path(inbox_path) / "labels"
if labels_dir.is_dir() and any(labels_dir.iterdir()):
stage = "returned"
result = register_batch(None, project, task, batch_name, stage=stage, location="inbox")
return {"ok": True, "delivery_id": delivery_id, "batch": result.get("batch")}

View File

@@ -24,26 +24,43 @@ def _new_delivery_id() -> str:
def _normalize_task(project: str, task: str | None) -> str | None:
if project == "dms":
return (task or "").strip() or None
return None
t = (task or "").strip() or None
if project == "adas":
return t or "cuboid_7cls"
if project == "lane":
return t
return t
def _enrich_delivery_dict(db, rec: BatchDelivery) -> dict[str, Any]:
def _enrich_delivery_dict(db, rec: BatchDelivery, *, ap_map: dict | None = None, job_map: dict | None = None) -> dict[str, Any]:
d = rec.to_dict()
d["submitted_by"] = rec.submitted_by_name
if rec.approval_id:
ap = db.get(Approval, rec.approval_id)
ap = (ap_map or {}).get(rec.approval_id) if ap_map is not None else db.get(Approval, rec.approval_id)
if ap:
d["approval_status"] = ap.status
d["job_id"] = ap.job_id
if ap.job_id:
job = db.get(Job, ap.job_id)
job = (job_map or {}).get(ap.job_id) if job_map is not None else db.get(Job, ap.job_id)
if job:
d["job_status"] = job.status
return d
def _bulk_approval_maps(db, rows: list[BatchDelivery]) -> tuple[dict[str, Approval], dict[str, Job]]:
approval_ids = [r.approval_id for r in rows if r.approval_id]
ap_map: dict[str, Approval] = {}
job_map: dict[str, Job] = {}
if approval_ids:
aps = db.query(Approval).filter(Approval.id.in_(approval_ids)).all()
ap_map = {a.id: a for a in aps}
job_ids = [a.job_id for a in aps if a.job_id]
if job_ids:
jobs = db.query(Job).filter(Job.id.in_(job_ids)).all()
job_map = {j.id: j for j in jobs}
return ap_map, job_map
def list_deliveries(
*,
status: str | None = None,
@@ -65,8 +82,9 @@ def list_deliveries(
q = q.filter(BatchDelivery.status.in_(("draft", "rejected", "ingest_failed")))
total = q.count()
rows = q.offset(max(0, offset)).limit(max(1, limit)).all()
ap_map, job_map = _bulk_approval_maps(db, rows)
return {
"items": [_enrich_delivery_dict(db, r) for r in rows],
"items": [_enrich_delivery_dict(db, r, ap_map=ap_map, job_map=job_map) for r in rows],
"total": total,
"offset": offset,
"limit": limit,

View File

@@ -377,9 +377,14 @@ def get_live_fleet(db: Session) -> dict[str, Any]:
items = []
for v in vehicles:
active = get_active_run(db, v.id)
row = v.to_dict()
row["vehicle_id"] = row["id"]
row["lat"] = row.get("last_lat")
row["lng"] = row.get("last_lng")
row["speed_kmh"] = row.get("last_speed_kmh")
items.append(
{
**v.to_dict(),
**row,
"active_run_id": active.id if active else None,
"active_mileage_km": active.mileage_km if active else None,
"active_run_no": active.run_no if active else None,

View File

@@ -35,6 +35,15 @@ def validate_delivery_fields(
return f"数据路径不存在: {path}"
if project == "dms" and p.name in ("train", "val", "test") and p.parent.name == "images":
return f"请填批次根目录(例如 {p.parent.parent}),不要填到 images/train"
if project == "adas" and task == "det_7cls":
if "/inbox/det_7cls/" not in str(p).replace("\\", "/"):
return "ADAS 2D 须落在 adas/inbox/det_7cls/{批次}project=adas, task=det_7cls"
if project == "adas" and task in (None, "", "cuboid_7cls"):
if "/inbox/cuboid_7cls/" not in str(p).replace("\\", "/"):
return "ADAS 3D 须落在 adas/inbox/cuboid_7cls/{批次}project=adas, task=cuboid_7cls"
if project == "dms" and task == "adas":
if "/inbox/adas/" not in str(p).replace("\\", "/"):
return "旧版 ADAS 2D 路径 dms/inbox/adas/{批次};新数据请用 adas/inbox/det_7cls/"
return None
@@ -59,7 +68,7 @@ def ingest_from_directory(
raise ValueError(err)
src = Path(data_path.strip())
task_eff = task if project == "dms" else None
task_eff = task if project in ("dms", "adas") else None
cand = create_directory_candidate(
project=project,
task=task_eff,
@@ -117,4 +126,11 @@ def run_delivery_ingest(delivery_id: str) -> dict[str, Any]:
rec.error_message = None
db.flush()
try:
from as_platform.deliveries.scan import bridge_delivery_to_workbench
bridge_delivery_to_workbench(delivery_id)
except Exception:
pass
return result

View File

@@ -0,0 +1,294 @@
"""批次索引:扫盘结果落库,列表页只查 DB<200ms"""
from __future__ import annotations
import time
from datetime import datetime, timezone
from typing import Any
from as_platform.data.core import get_pending_report, load_wf
from as_platform.db.engine import session_scope
from as_platform.db.models import BatchIndex, LabelingCampaign
from as_platform.labeling.scope import enrich_batch_labels, load_dms_registry
from as_platform.labeling.stage import STAGE_ALIASES, effective_stage
def _utcnow() -> datetime:
return datetime.now(timezone.utc)
def campaign_id_for_row(project: str, task: str, mode: str | None, batch: str, location: str) -> str:
from as_platform.labeling.service import _campaign_id
return _campaign_id(project, task, mode, batch, location)
def _batch_dict_to_fields(b: dict[str, Any], reg: dict) -> dict[str, Any] | None:
if b.get("registry_only") and not b.get("batch"):
return None
row = enrich_batch_labels(b, reg)
if row.get("registry_only"):
pass
raw_stage = row.get("stage")
eff = effective_stage(raw_stage) or raw_stage or "raw_pool"
allowed = (
"raw_pool", "out_for_labeling", "returned", "labeling_submitted",
"in_review", "review_approved", "review_rejected", "ingested",
)
if eff not in allowed and raw_stage not in allowed:
return None
project = row.get("project") or "dms"
task = row.get("task") or ""
mode = row.get("mode")
batch = row.get("batch") or ""
location = row.get("location") or "inbox"
counts = row.get("counts") or {}
cid = campaign_id_for_row(project, task, mode, batch, location)
return {
"campaign_id": cid,
"project": project,
"task": task or None,
"mode": mode,
"batch": batch,
"pack": row.get("pack"),
"location": location,
"stage": eff,
"batch_path": row.get("path"),
"scope_key": row.get("scope_key"),
"engineer": row.get("engineer"),
"format": row.get("format"),
"image_count": int(counts.get("images") or 0),
"label_count": int(counts.get("labels") or 0),
"has_meta": bool(row.get("has_meta")),
"registry_only": bool(row.get("registry_only")),
"next_cli": row.get("next_cli"),
"domain": row.get("domain"),
"domain_label": row.get("domain_label"),
"task_label": row.get("task_label"),
"mode_label": row.get("mode_label"),
"labeling_profile": row.get("labeling_profile"),
"export_default": row.get("export_default"),
"ml_adapter": row.get("ml_adapter"),
"indexed_at": _utcnow(),
}
def upsert_batch_dict(b: dict[str, Any], *, reg: dict | None = None) -> str | None:
"""登记/单批更新时写入索引。"""
reg = reg or load_dms_registry()
fields = _batch_dict_to_fields(b, reg)
if not fields:
return None
cid = fields["campaign_id"]
fields["archived"] = False
with session_scope() as db:
rec = db.get(BatchIndex, cid)
if rec:
for k, v in fields.items():
setattr(rec, k, v)
else:
db.add(BatchIndex(**fields))
db.commit()
return cid
def archive_batch(campaign_id: str) -> dict[str, Any]:
"""软删除:仅从工作台隐藏,不删磁盘数据。"""
with session_scope() as db:
rec = db.get(BatchIndex, campaign_id)
if not rec:
raise FileNotFoundError(campaign_id)
if rec.archived:
return {"ok": True, "campaign_id": campaign_id, "already_archived": True}
if rec.stage != "raw_pool":
raise ValueError("仅「待送标」批次可移除")
camp = db.get(LabelingCampaign, campaign_id)
if camp and camp.status not in ("not_opened", ""):
raise ValueError("已开标的批次不可移除,请先在标注进度中处理")
rec.archived = True
rec.indexed_at = _utcnow()
db.commit()
return {"ok": True, "campaign_id": campaign_id}
def sync_index_stage(campaign_id: str, stage: str) -> None:
eff = effective_stage(stage) or stage
with session_scope() as db:
rec = db.get(BatchIndex, campaign_id)
if rec:
rec.stage = eff
rec.indexed_at = _utcnow()
db.commit()
def index_count() -> int:
with session_scope() as db:
return db.query(BatchIndex).count()
def index_is_empty() -> bool:
return index_count() == 0
def get_batch_by_campaign_id(campaign_id: str) -> dict[str, Any] | None:
with session_scope() as db:
rec = db.get(BatchIndex, campaign_id)
return rec.to_list_row() if rec else None
def rebuild_batch_index(wf: dict | None = None) -> dict[str, Any]:
"""全量扫盘并重建索引(刷新/首次加载/登记后批量同步)。"""
from as_platform.labeling.service import _registry_fallback_batches
t0 = time.perf_counter()
wf = wf or load_wf()
reg = load_dms_registry()
report = get_pending_report(wf)
candidates: list[dict[str, Any]] = list(report.get("batches") or [])
candidates.extend(_registry_fallback_batches(wf, reg))
seen: set[str] = set()
upserted = 0
with session_scope() as db:
for b in candidates:
fields = _batch_dict_to_fields(b, reg)
if not fields:
continue
cid = fields["campaign_id"]
seen.add(cid)
rec = db.get(BatchIndex, cid)
if rec and rec.archived:
continue
if rec:
for k, v in fields.items():
setattr(rec, k, v)
else:
db.add(BatchIndex(**fields))
upserted += 1
if seen:
db.query(BatchIndex).filter(
BatchIndex.campaign_id.notin_(seen),
BatchIndex.archived.is_(False),
).delete(synchronize_session=False)
db.commit()
elapsed_ms = round((time.perf_counter() - t0) * 1000)
return {
"ok": True,
"count": upserted,
"elapsed_ms": elapsed_ms,
"updated_at": report.get("updated_at"),
}
def _expand_stage_filters_for_sql(stage_filters: list[str]) -> list[str]:
"""将筛选阶段展开为索引表中的实际 stage 值(含 review_approved 等别名)。"""
expanded: set[str] = set()
for sf in stage_filters:
expanded.add(sf)
for alias, canonical in STAGE_ALIASES.items():
if canonical == sf:
expanded.add(alias)
return list(expanded)
def _index_query(db, *, stage_filters: list[str], q: str | None):
from sqlalchemy import func, or_
from as_platform.config import IS_POSTGRES
query = db.query(BatchIndex).filter(
BatchIndex.registry_only.is_(False),
BatchIndex.archived.is_(False),
)
if stage_filters:
query = query.filter(BatchIndex.stage.in_(_expand_stage_filters_for_sql(stage_filters)))
text = (q or "").strip()
if text:
pattern = f"%{text}%"
if IS_POSTGRES:
query = query.filter(
or_(
BatchIndex.batch.ilike(pattern),
BatchIndex.task.ilike(pattern),
BatchIndex.project.ilike(pattern),
)
)
else:
lp = pattern.lower()
query = query.filter(
or_(
func.lower(BatchIndex.batch).like(lp),
func.lower(func.coalesce(BatchIndex.task, "")).like(lp),
func.lower(BatchIndex.project).like(lp),
)
)
return query
def list_batches_from_index(
*,
stage: str | None = None,
stages: list[str] | None = None,
offset: int = 0,
limit: int = 20,
q: str | None = None,
) -> dict[str, Any]:
stage_filters = [s.strip() for s in (stages or []) if s and s.strip()]
if stage and stage not in stage_filters:
stage_filters.append(stage)
with session_scope() as db:
base = _index_query(db, stage_filters=stage_filters, q=q)
total = base.count()
recs = (
base.order_by(BatchIndex.indexed_at.desc())
.offset(max(0, offset))
.limit(max(1, limit))
.all()
)
page = [rec.to_list_row() for rec in recs]
latest_indexed = max((rec.indexed_at for rec in recs), default=None)
cids = [r["campaign_id"] for r in page if r.get("campaign_id")]
camp_map: dict[str, dict[str, Any]] = {}
if cids:
with session_scope() as db:
camps = db.query(LabelingCampaign).filter(LabelingCampaign.id.in_(cids)).all()
for c in camps:
camp_map[c.id] = {
"status": c.status,
"assigned_to_user_id": c.assigned_to_user_id,
"assigned_to_name": c.assigned_to_name,
}
progress_stages = frozenset({"out_for_labeling", "in_progress"})
out: list[dict[str, Any]] = []
for row in page:
cid = row.get("campaign_id")
camp = camp_map.get(cid) if cid else None
status = camp["status"] if camp else "not_opened"
if camp:
row["assigned_to_user_id"] = camp["assigned_to_user_id"]
row["assigned_to_name"] = camp["assigned_to_name"]
row["campaign_status"] = status
eff_stage = row.get("stage") or ""
if camp and (status in progress_stages or eff_stage in progress_stages):
try:
from as_platform.labeling.progress import campaign_progress_summary
row.update(campaign_progress_summary(cid))
except Exception:
row.update({"total_tasks": 0, "completed_tasks": 0, "assigned_tasks": 0})
out.append(row)
updated_at = latest_indexed.isoformat() if latest_indexed else None
return {
"items": out,
"total": total,
"offset": offset,
"limit": limit,
"updated_at": updated_at,
"source": "index",
}

View File

@@ -70,6 +70,12 @@ def update_campaign_batch_meta_stage(camp: LabelingCampaign, stage: str) -> bool
if camp.mode:
meta.setdefault("mode", camp.mode)
write_meta(batch_dir, meta)
try:
from as_platform.labeling.batch_index import sync_index_stage
sync_index_stage(camp.id, stage)
except Exception:
pass
return True

View File

@@ -107,7 +107,10 @@ def _labels_from_registry_profile(project: str, task: str, mode: str | None) ->
prof = (load_labeling_registry().get("profiles") or {}).get(pk) or {}
cvat_names = prof.get("cvat_labels")
if cvat_names:
return [{"name": str(n), "type": "cuboid"} for n in cvat_names]
label_type = prof.get("cvat_label_type") or (
"cuboid" if project == "adas" and task == "cuboid_7cls" else "rectangle"
)
return [{"name": str(n), "type": label_type} for n in cvat_names]
return None
@@ -126,6 +129,10 @@ def build_cvat_labels(
if project == "adas":
if task == "cuboid_7cls":
return ADAS_CUBOID_7CLS_LABELS
if task == "det_7cls":
return [_rect_label(n) for n in [
"pedestrian", "car", "truck", "bus", "motorcycle", "tricycle", "traffic cone",
]]
return ADAS_CUBOID_7CLS_LABELS
if project == "lane":
@@ -145,4 +152,8 @@ def resolve_annotation_types(project: str, task: str | None = None, mode: str |
}
if project == "adas" and task == "cuboid_7cls":
return ["cuboid"]
if project == "adas" and task == "det_7cls":
return ["bbox"]
if project == "dms" and task == "adas":
return ["bbox"]
return mapping.get(project, ["bbox"])

View File

@@ -108,66 +108,23 @@ def _registry_fallback_batches(wf: dict, reg: dict) -> list[dict[str, Any]]:
def list_labeling_batches(
*,
stage: str | None = None,
stages: list[str] | None = None,
offset: int = 0,
limit: int = 20,
refresh: bool = False,
q: str | None = None,
) -> dict[str, Any]:
wf = load_wf()
report = get_pending_report(wf)
reg = load_dms_registry()
items: list[dict[str, Any]] = []
seen: set[str] = set()
allowed_stages = ("raw_pool", "out_for_labeling", "returned", "labeling_submitted", "in_review", "review_approved", "review_rejected")
from as_platform.labeling.batch_index import (
index_is_empty,
list_batches_from_index,
rebuild_batch_index,
)
def _append(b: dict[str, Any]) -> None:
if b.get("registry_only"):
return
raw_stage = b.get("stage")
eff = effective_stage(raw_stage)
if stage and not matches_stage_filter(raw_stage, stage):
return
if eff not in allowed_stages and raw_stage not in allowed_stages:
return
row = enrich_batch_labels(b, reg)
row["stage"] = eff or raw_stage
cid = _campaign_id(
row["project"], row.get("task") or "", row.get("mode"), row["batch"], row.get("location") or "inbox"
)
key = f"{cid}"
if key in seen:
return
seen.add(key)
with session_scope() as db:
camp = db.get(LabelingCampaign, cid)
status = camp.status if camp else "not_opened"
if camp:
row["assigned_to_user_id"] = camp.assigned_to_user_id
row["assigned_to_name"] = camp.assigned_to_name
row["campaign_id"] = cid
row["campaign_status"] = status
if camp and status in ("in_progress", "labeling_submitted"):
try:
from as_platform.labeling.progress import campaign_progress_summary
row.update(campaign_progress_summary(cid))
except Exception:
row.update({"total_tasks": 0, "completed_tasks": 0, "assigned_tasks": 0})
items.append(row)
for b in report.get("batches", []):
_append(b)
for b in _registry_fallback_batches(wf, reg):
_append(b)
total = len(items)
page = items[max(0, offset) : max(0, offset) + max(1, limit)]
return {
"items": page,
"total": total,
"offset": offset,
"limit": limit,
"updated_at": report.get("updated_at"),
}
if refresh or index_is_empty():
rebuild_batch_index()
return list_batches_from_index(
stage=stage, stages=stages, offset=offset, limit=limit, q=q,
)
def open_campaign(
@@ -189,13 +146,14 @@ def open_campaign(
cvat = get_cvat_client()
if not cvat.ping():
raise ValueError("CVAT 标注引擎不可用,请执行: docker compose -f docker-compose.yml -f docker-compose.cvat.yml up -d")
raise ValueError("CVAT 标注引擎不可用,请执行: docker compose up -d")
cvat_labels = build_cvat_labels(project, task, mode, ann_types)
cvat_task = cvat.create_task(name=cid, labels=cvat_labels)
cvat_task_id = cvat_task.id
cvat_job_url = cvat_task.job_url
batch_dir = None
with session_scope() as db:
camp = db.get(LabelingCampaign, cid)
if not camp:
@@ -241,6 +199,26 @@ def open_campaign(
reg = load_dms_registry() if project == "dms" else None
row = enrich_batch_labels(out, reg)
row["stage"] = "out_for_labeling"
try:
from as_platform.labeling.batch_index import upsert_batch_dict
if batch_dir and batch_dir.is_dir():
from as_platform.data.batch import enrich_batch
upsert_batch_dict(
enrich_batch(
batch_dir,
project=project,
task=task,
pack=pack,
batch=batch,
location=location,
),
)
else:
upsert_batch_dict(row)
except Exception:
pass
return row
@@ -376,23 +354,15 @@ def list_labeling_assignees() -> dict[str, Any]:
def _find_batch_for_campaign_id(campaign_id: str) -> dict[str, Any] | None:
"""确定性 campaign_id 反查 pending / registry 批次行。"""
wf = load_wf()
reg = load_dms_registry()
candidates: list[dict[str, Any]] = []
report = get_pending_report(wf)
candidates.extend(report.get("batches") or [])
candidates.extend(_registry_fallback_batches(wf, reg))
for b in candidates:
cid = _campaign_id(
b.get("project") or "dms",
b.get("task") or "",
b.get("mode"),
b.get("batch") or "",
b.get("location") or "inbox",
)
if cid == campaign_id:
return b
"""由 campaign_id 反查批次行(优先读索引)"""
from as_platform.labeling.batch_index import get_batch_by_campaign_id, index_is_empty, rebuild_batch_index
row = get_batch_by_campaign_id(campaign_id)
if row:
return row
if index_is_empty():
rebuild_batch_index()
return get_batch_by_campaign_id(campaign_id)
return None

View File

@@ -0,0 +1,280 @@
#!/usr/bin/env python3
"""DMS 2 图 E2E标完后自动 提交→质检→导出→build 入库。"""
from __future__ import annotations
import argparse
import hashlib
import json
import sys
import time
import urllib.error
import urllib.request
from pathlib import Path
from typing import Any
ROOT = Path(__file__).resolve().parents[3]
PLATFORM = ROOT / "platform"
if str(PLATFORM) not in sys.path:
sys.path.insert(0, str(PLATFORM))
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
def campaign_id(project: str, task: str, mode: str | None, batch: str, location: str = "inbox") -> str:
from as_platform.labeling.scope import format_scope_key
sk = format_scope_key(project, task, mode)
raw = f"{sk}:{batch}:{location}"
return hashlib.sha256(raw.encode()).hexdigest()[:20]
class ApiClient:
def __init__(self, base: str, token: str) -> None:
self.base = base.rstrip("/")
self.token = token
def _request(self, method: str, path: str, body: dict | None = None) -> Any:
data = json.dumps(body).encode() if body is not None else None
req = urllib.request.Request(
f"{self.base}{path}",
data=data,
headers={
"Authorization": f"Bearer {self.token}",
"Content-Type": "application/json",
},
method=method,
)
try:
with urllib.request.urlopen(req, timeout=120) as resp:
raw = resp.read().decode()
return json.loads(raw) if raw else {}
except urllib.error.HTTPError as e:
detail = e.read().decode()
raise RuntimeError(f"{method} {path} -> {e.code}: {detail}") from e
def get(self, path: str) -> Any:
return self._request("GET", path)
def post(self, path: str, body: dict | None = None) -> Any:
return self._request("POST", path, body)
def login(base: str, name: str = "e2e-runner") -> ApiClient:
req = urllib.request.Request(
f"{base.rstrip('/')}/api/v1/auth/dev/login",
data=json.dumps({"name": name}).encode(),
headers={"Content-Type": "application/json"},
method="POST",
)
with urllib.request.urlopen(req, timeout=30) as resp:
data = json.loads(resp.read().decode())
return ApiClient(base, data["access_token"])
def wait_job(api: ApiClient, job_id: str, timeout: int = 180) -> dict[str, Any]:
deadline = time.time() + timeout
while time.time() < deadline:
job = api.get(f"/api/v1/jobs/{job_id}")
st = job.get("status")
if st in ("succeeded", "failed"):
return job
time.sleep(1)
raise TimeoutError(f"job {job_id} not finished in {timeout}s")
def labeled_count(batch_dir: Path) -> int:
ann_dir = batch_dir / "labels" / "ls_annotations"
if not ann_dir.is_dir():
return 0
from as_platform.labeling.progress import _annotation_has_result
n = 0
for p in ann_dir.glob("*.json"):
if _annotation_has_result(p):
n += 1
return n
def batch_stage(batch_dir: Path) -> str:
meta = batch_dir / "batch.meta.yaml"
if not meta.is_file():
return "raw_pool"
import yaml
data = yaml.safe_load(meta.read_text(encoding="utf-8")) or {}
return str(data.get("stage") or "raw_pool")
def cmd_info(args: argparse.Namespace) -> None:
cid = campaign_id(args.project, args.task, None, args.batch)
print(f"campaign_id={cid}")
print(f"annotate_url=/labeling/annotate/{cid}")
print(f"batch_path=datasets/dms/inbox/{args.task}/{args.batch}")
wf_root = ROOT / "datasets" / "dms"
batch_dir = wf_root / "inbox" / args.task / args.batch
if batch_dir.is_dir():
print(f"stage={batch_stage(batch_dir)} labeled={labeled_count(batch_dir)}")
try:
api = login(args.api)
row = next(
(i for i in api.get("/api/v1/labeling/batches?limit=100").get("items", [])
if i.get("batch") == args.batch and i.get("task") == args.task),
None,
)
if row:
print(f"platform_stage={row.get('stage')} status={row.get('campaign_status')}")
print(f"progress={row.get('completed_tasks', '?')}/{row.get('total_tasks', '?')}")
except Exception as e:
print(f"api_skip={e}")
def cmd_setup(args: argparse.Namespace) -> None:
api = login(args.api)
body = {
"project": args.project,
"task": args.task,
"batch": args.batch,
"location": "inbox",
}
row = api.post("/api/v1/labeling/campaigns/open", body)
print(json.dumps({"campaign_id": row.get("id"), "stage": row.get("stage"), "cvat_task_id": row.get("cvat_task_id")}, ensure_ascii=False))
def wait_labels(batch_dir: Path, min_images: int, wait_sec: int) -> None:
if labeled_count(batch_dir) >= min_images:
return
if wait_sec <= 0:
raise RuntimeError(
f"仅标注 {labeled_count(batch_dir)}/{min_images} 张,请先在平台画框保存,再执行 run 或 run-wait"
)
print(f"等待标注 {min_images} 张 (最多 {wait_sec}s)...")
deadline = time.time() + wait_sec
while time.time() < deadline:
n = labeled_count(batch_dir)
if n >= min_images:
print(f"labeled={n}")
return
time.sleep(3)
raise TimeoutError(f"超时:仅 {labeled_count(batch_dir)}/{min_images} 张有标注")
def cmd_run(args: argparse.Namespace) -> None:
cid = campaign_id(args.project, args.task, None, args.batch)
batch_dir = ROOT / "datasets" / "dms" / "inbox" / args.task / args.batch
if not batch_dir.is_dir():
raise FileNotFoundError(batch_dir)
wait_labels(batch_dir, args.min_images, args.wait_label_sec)
api = login(args.api)
print("==> 1. 提交质检")
api.post(f"/api/v1/labeling/campaigns/{cid}/submit")
row = next(
i for i in api.get("/api/v1/labeling/batches?limit=100").get("items", [])
if i.get("campaign_id") == cid
)
assert row.get("stage") == "in_review", row
print("==> 2. 质检通过 (全部 good)")
queue = api.get(f"/api/v1/labeling/campaigns/{cid}/review-queue?limit=50")
items = queue.get("items") or []
scores = [{"image_path": it["image_path"], "score": "good"} for it in items]
res = api.post(
f"/api/v1/labeling/campaigns/{cid}/review-submit",
{"scores": scores},
)
print("review", res)
row = next(
i for i in api.get("/api/v1/labeling/batches?limit=100").get("items", [])
if i.get("campaign_id") == cid
)
assert row.get("stage") == "labeling_submitted", row
print("==> 3. 执行导出")
exp = api.post(f"/api/v1/labeling/campaigns/{cid}/export")
job_id = (exp.get("job") or {}).get("id")
assert job_id, exp
job = wait_job(api, job_id)
if job.get("status") != "succeeded":
raise RuntimeError(f"export failed: {job}")
print("export_job", job.get("result"))
row = next(
i for i in api.get("/api/v1/labeling/batches?limit=100").get("items", [])
if i.get("campaign_id") == cid
)
assert row.get("stage") == "returned", row
yolo = list((batch_dir / "labels").rglob("*.txt"))
assert yolo, "export 后应有 YOLO txt"
print("==> 4. 提交 build 审核")
appr = api.post(
"/api/v1/system/audit/submit-build-batch",
{
"project": args.project,
"task": args.task,
"batch": args.batch,
"pack": args.pack,
"location": "inbox",
"note": f"E2E smoke {args.batch}",
},
)
approval_id = appr.get("id")
assert approval_id, appr
print("approval_id", approval_id)
print("==> 5. 批准 build")
done = api.post(f"/api/v1/system/audit/{approval_id}/approve", {"comment": "e2e auto approve"})
build_job_id = done.get("job_id")
if build_job_id:
bjob = wait_job(api, build_job_id, timeout=300)
if bjob.get("status") != "succeeded":
raise RuntimeError(f"build failed: {bjob}")
print("build_job", bjob.get("result"))
row = next(
i for i in api.get("/api/v1/labeling/batches?limit=100").get("items", [])
if i.get("campaign_id") == cid
)
assert row.get("stage") == "ingested", row
assert batch_stage(batch_dir) == "ingested", batch_stage(batch_dir)
dest = ROOT / "datasets" / "dms" / "packs" / args.pack / args.task / "sources" / args.batch
assert dest.is_dir(), f"missing pack source: {dest}"
dest_labels = list(dest.rglob("labels/**/*.txt")) + list(dest.rglob("labels/*.txt"))
assert dest_labels, f"pack 内应有 labels: {dest}"
print("DMS_E2E_PIPELINE_OK")
print(json.dumps({
"campaign_id": cid,
"batch": args.batch,
"pack": args.pack,
"dest": str(dest),
"yolo_in_inbox": len(yolo),
"stage": row.get("stage"),
}, ensure_ascii=False, indent=2))
def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("command", choices=("setup", "run", "info"))
ap.add_argument("--api", default="http://127.0.0.1:8787")
ap.add_argument("--project", default="dms")
ap.add_argument("--task", default="addw")
ap.add_argument("--batch", default="e2e_2img_20260616")
ap.add_argument("--pack", default="dms_v1")
ap.add_argument("--min-images", type=int, default=2)
ap.add_argument("--wait-label-sec", type=int, default=0)
ap.add_argument("--skip-files", action="store_true")
args = ap.parse_args()
if args.command == "info":
cmd_info(args)
elif args.command == "setup":
cmd_setup(args)
elif args.command == "run":
cmd_run(args)
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,54 @@
"""批次索引:列表走 DB重建走扫盘。"""
from __future__ import annotations
import time
from as_platform.labeling.batch_index import (
index_is_empty,
list_batches_from_index,
rebuild_batch_index,
)
from as_platform.labeling.service import list_labeling_batches
def test_rebuild_and_list_from_index():
r = rebuild_batch_index()
assert r["ok"] is True
assert r["count"] >= 0
t0 = time.perf_counter()
out = list_batches_from_index(limit=100)
elapsed_ms = (time.perf_counter() - t0) * 1000
assert out["source"] == "index"
assert "items" in out
assert elapsed_ms < 500, f"index list too slow: {elapsed_ms:.0f}ms"
def test_list_labeling_batches_uses_index():
if index_is_empty():
rebuild_batch_index()
t0 = time.perf_counter()
out = list_labeling_batches(limit=50)
elapsed_ms = (time.perf_counter() - t0) * 1000
assert "items" in out
assert elapsed_ms < 800, f"list_labeling_batches too slow: {elapsed_ms:.0f}ms"
def test_archive_batch_hides_from_list():
from as_platform.db.engine import session_scope
from as_platform.db.models import BatchIndex
from as_platform.labeling.batch_index import archive_batch, list_batches_from_index
with session_scope() as db:
rec = (
db.query(BatchIndex)
.filter(BatchIndex.archived.is_(False), BatchIndex.stage == "raw_pool")
.first()
)
if not rec:
return
cid = rec.campaign_id
archive_batch(cid)
out = list_batches_from_index(stage="raw_pool", limit=500)
assert all(r.get("campaign_id") != cid for r in out["items"])